diff --git "a/wandb/run-20220830_110431-yvlr8ud4/logs/debug-internal.log" "b/wandb/run-20220830_110431-yvlr8ud4/logs/debug-internal.log" --- "a/wandb/run-20220830_110431-yvlr8ud4/logs/debug-internal.log" +++ "b/wandb/run-20220830_110431-yvlr8ud4/logs/debug-internal.log" @@ -1,67816 +1,3 @@ -2022-08-30 11:04:32,108 INFO MainThread:3760704 [internal.py:wandb_internal():90] W&B internal server running at pid: 3760704, started at: 2022-08-30 11:04:32.108122 -2022-08-30 11:04:32,110 INFO WriterThread:3760704 [datastore.py:open_for_write():75] open: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/run-yvlr8ud4.wandb -2022-08-30 11:04:32,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: header -2022-08-30 11:04:32,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: run -2022-08-30 11:04:32,384 INFO SenderThread:3760704 [dir_watcher.py:__init__():166] watching files in: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files -2022-08-30 11:04:32,384 INFO SenderThread:3760704 [sender.py:_start_run_threads():811] run started: yvlr8ud4 with start time 1661850271 -2022-08-30 11:04:32,384 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 11:04:32,384 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 11:04:32,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: check_version -2022-08-30 11:04:32,386 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: check_version -2022-08-30 11:04:32,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: run_start -2022-08-30 11:04:32,564 DEBUG HandlerThread:3760704 [meta.py:__init__():35] meta init -2022-08-30 11:04:32,564 DEBUG HandlerThread:3760704 [meta.py:__init__():49] meta init done -2022-08-30 11:04:32,564 DEBUG HandlerThread:3760704 [meta.py:probe():209] probe -2022-08-30 11:04:32,565 DEBUG HandlerThread:3760704 [meta.py:_setup_git():199] setup git -2022-08-30 11:04:32,581 DEBUG HandlerThread:3760704 [meta.py:_setup_git():206] setup git done -2022-08-30 11:04:32,582 DEBUG HandlerThread:3760704 [meta.py:_save_pip():53] save pip -2022-08-30 11:04:32,582 DEBUG HandlerThread:3760704 [meta.py:_save_pip():67] save pip done -2022-08-30 11:04:32,582 DEBUG HandlerThread:3760704 [meta.py:probe():247] probe done -2022-08-30 11:04:32,591 DEBUG SenderThread:3760704 [sender.py:send():232] send: files -2022-08-30 11:04:32,591 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-metadata.json with policy now -2022-08-30 11:04:32,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:04:32,599 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:04:32,870 DEBUG SenderThread:3760704 [sender.py:send():232] send: telemetry -2022-08-30 11:04:32,870 DEBUG SenderThread:3760704 [sender.py:send():232] send: config -2022-08-30 11:04:32,872 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:04:32,872 DEBUG SenderThread:3760704 [sender.py:send():232] send: telemetry -2022-08-30 11:04:32,872 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:04:32,872 WARNING SenderThread:3760704 [sender.py:send_metric():904] Seen metric with glob (shouldn't happen) -2022-08-30 11:04:32,872 DEBUG SenderThread:3760704 [sender.py:send():232] send: telemetry -2022-08-30 11:04:33,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_created():213] file/dir created: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:04:33,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_created():213] file/dir created: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 11:04:33,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_created():213] file/dir created: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/requirements.txt -2022-08-30 11:04:33,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_created():213] file/dir created: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-metadata.json -2022-08-30 11:04:34,428 INFO Thread-11 :3760704 [upload_job.py:push():137] Uploaded file /tmp/tmpqnei5jrkwandb/26n8mx0t-wandb-metadata.json -2022-08-30 11:04:35,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:04:41,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:04:45,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:04:47,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:04:47,946 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:04:49,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:04:53,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:04:55,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:04:59,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:00,643 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:05:03,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:05:03,166 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:05:03,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:03,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/config.yaml -2022-08-30 11:05:05,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:09,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:11,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:13,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:17,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:18,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:05:18,459 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:05:19,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:21,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:25,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:27,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:29,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:30,735 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:05:31,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:33,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:33,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:05:33,802 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:05:35,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:39,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:41,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:43,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:45,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:47,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:49,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:05:49,062 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:05:49,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:51,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:53,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:56,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:05:58,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:00,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:00,810 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:06:02,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:04,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:06:04,336 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:06:04,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:06,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:08,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:10,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:12,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:14,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:16,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:18,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:19,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:06:19,557 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:06:20,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:20,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:20,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:22,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:25,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:25,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:27,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:28,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:29,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:29,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:30,885 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:06:31,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:31,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:32,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:33,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:33,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:34,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:06:34,872 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:06:36,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:36,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:37,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:37,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:39,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:06:40,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:44,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:46,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:50,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:06:50,136 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:06:50,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:52,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:56,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:06:58,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:00,962 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:07:02,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:04,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:05,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:07:05,358 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:07:08,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:10,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:12,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:14,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:16,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:20,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:20,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:07:20,642 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:07:22,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:24,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:26,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:28,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:30,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:31,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:07:32,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:34,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:35,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:07:35,854 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:07:36,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:38,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:40,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:42,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:44,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:46,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:48,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:50,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:51,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:07:51,147 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:07:52,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:54,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:56,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:07:58,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:00,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:01,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:08:02,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:04,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:06,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:08:06,392 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:08:06,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:06,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:06,915 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,920 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,926 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,931 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,936 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,942 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,947 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,956 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,961 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,961 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,967 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,972 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,978 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,983 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,989 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:06,994 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,000 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,005 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,007 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,008 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,008 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,008 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,008 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,008 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,008 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,009 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,009 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,009 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,009 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,009 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,009 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,009 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,010 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,010 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,010 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,010 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,010 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,010 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,010 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,011 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,011 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,011 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,011 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,011 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,011 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,011 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,012 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,012 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,012 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,012 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,012 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,012 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,012 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,013 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,013 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,013 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,013 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,013 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,013 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,013 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,014 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,014 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,014 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,014 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,014 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,014 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,014 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,015 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,015 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,015 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,015 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,015 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,015 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,015 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,016 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,016 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,016 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,016 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,016 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,016 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,016 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,017 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,017 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,017 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,017 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,017 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,017 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,017 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,018 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,018 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,018 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,018 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,018 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,018 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,018 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,019 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,019 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,019 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,019 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,019 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,019 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,019 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,020 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,020 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,020 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,020 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,020 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,020 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,020 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,021 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,021 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,021 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,021 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,021 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,021 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,021 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,022 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,022 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,022 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,022 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,022 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,022 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,022 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,022 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,023 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,023 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,023 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,023 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,023 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,023 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,023 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,023 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,023 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,023 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,023 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,027 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,027 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,027 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,027 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,027 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,027 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,027 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,027 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,027 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,027 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,031 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,035 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,039 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,042 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,046 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,047 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,050 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,054 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,056 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,057 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,068 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,071 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,072 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,078 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,079 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,080 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,084 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,087 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,091 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,092 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,095 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,096 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,098 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,099 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,101 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,104 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,109 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,118 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,118 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,118 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 11:08:07,118 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 11:08:07,174 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 11:08:07,230 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 11:08:07,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 11:08:08,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:09,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:13,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:13,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:15,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:15,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:16,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:17,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:19,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:19,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:20,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:08:21,645 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:08:21,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:21,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:22,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:23,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:23,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:25,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:25,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:27,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:28,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:29,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:29,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:30,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:31,292 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:08:32,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:32,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:33,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:34,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:34,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:35,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:36,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/config.yaml -2022-08-30 11:08:36,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:08:36,868 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:08:37,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:08:37,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:39,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:43,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:45,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:47,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:51,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:52,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:08:52,209 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:08:53,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:55,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:08:59,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:01,390 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:09:01,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:03,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:05,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:07,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:07,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:09:07,814 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:09:09,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:11,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:13,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:17,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:19,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:21,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:23,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:09:23,125 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:09:23,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:25,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:27,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:29,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:31,464 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:09:31,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:33,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:35,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:37,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:38,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:09:38,347 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:09:39,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:41,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:43,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:45,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:48,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:50,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:52,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:53,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:53,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:53,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:53,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:53,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:09:53,569 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:09:53,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:53,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:53,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:53,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:53,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:53,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:53,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:53,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:09:54,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:09:56,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:00,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:01,574 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:10:04,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:06,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:06,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:08,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:10:08,788 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:10:08,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:08,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:12,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:12,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:13,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:15,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:16,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:17,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:17,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:18,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:19,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:20,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:20,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:21,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:22,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:22,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:23,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:10:24,027 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:10:24,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:24,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:25,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:26,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:26,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:27,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:30,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:31,609 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:10:32,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:33,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:33,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:10:34,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:38,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:39,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:10:39,430 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:10:40,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:44,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:46,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:48,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:50,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:52,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:54,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:10:54,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:10:54,713 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:10:58,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:00,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:01,681 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:11:02,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:04,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:06,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:08,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:10,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:11:10,011 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:11:10,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:12,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:14,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:16,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:18,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:20,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:22,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:24,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:25,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:11:25,232 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:11:26,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:28,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:30,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:31,747 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:11:32,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:34,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:36,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:38,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:40,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:11:40,526 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:11:40,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:41,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:11:41,538 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 11:11:41,767 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 11:11:41,831 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 11:11:42,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:42,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 11:11:48,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:52,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:55,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:11:55,759 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:11:56,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:11:59,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:01,904 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:12:02,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:02,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:03,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:06,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:06,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:07,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:07,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:08,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:09,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:09,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:10,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:12:11,001 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:12:11,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:11,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:12,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:13,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:13,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:14,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:15,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:16,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:17,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:18,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:18,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:19,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:20,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:20,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:21,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:23,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:25,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:25,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:26,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:12:26,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:12:26,229 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:12:29,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:31,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:31,981 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:12:33,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:35,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:39,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:41,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:12:41,452 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:12:41,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:43,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:45,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:47,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:49,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:51,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:55,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:56,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:12:56,703 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:12:57,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:12:59,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:01,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:02,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:13:03,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:05,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:07,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:09,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:11,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:11,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:13:11,964 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:13:13,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:15,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:17,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:19,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:21,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:23,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:25,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:27,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:13:27,185 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:13:27,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:29,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:29,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:31,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:32,134 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:13:35,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:41,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:42,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:13:42,437 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:13:45,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:47,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:51,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:55,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:57,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:13:57,734 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:13:57,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:13:59,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:13:59,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:01,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:02,213 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:14:02,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:02,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:03,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:04,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:04,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:05,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:06,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:07,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:07,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:08,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:09,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:10,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:11,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:11,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:14:13,223 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:14:13,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:13,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:14,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:14,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:15,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:15,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:16,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:17,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:18,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:19,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:21,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:21,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:22,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:22,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:14:23,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:26,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:28,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:14:28,448 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:14:30,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:32,290 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:14:32,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:34,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:36,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:38,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:40,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:42,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:43,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:14:43,708 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:14:44,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:46,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:48,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:50,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:52,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:54,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:56,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:58,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:14:58,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:14:58,934 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:15:00,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:02,360 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:15:02,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:04,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:06,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:08,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:10,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:12,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:14,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:15:14,160 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:15:14,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:16,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:17,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:17,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:17,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:17,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:17,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:17,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:17,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:17,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:17,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:17,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:17,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:17,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:18,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:18,113 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 11:15:18,312 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 11:15:18,372 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 11:15:18,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:18,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 11:15:20,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:24,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:28,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:29,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:15:29,402 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:15:32,441 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:15:32,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:36,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:40,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:44,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:15:44,623 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:15:44,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:46,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:50,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:52,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:54,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:54,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:55,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:56,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:57,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:57,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:15:58,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:58,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:15:59,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:15:59,864 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:16:00,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:00,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:01,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,516 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:16:02,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:02,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:02,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:03,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:04,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:04,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:05,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:06,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:07,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:07,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:08,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:09,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:10,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:11,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:11,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:12,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:13,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:13,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:14,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:15,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:16:15,100 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:16:16,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:17,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:17,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:18,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:20,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:21,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:21,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:16:22,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:24,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:26,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:28,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:30,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:16:30,345 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:16:31,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:32,592 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:16:33,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:35,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:37,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:39,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:41,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:43,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:45,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:16:45,571 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:16:45,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:47,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:49,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:51,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:53,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:55,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:57,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:16:59,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:00,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:17:00,813 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:17:01,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:02,667 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:17:03,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:05,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:05,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:05,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:05,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:05,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:05,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:05,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:05,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:05,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:05,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:05,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:05,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:05,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:07,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:13,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:16,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:17:16,116 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:17:17,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:21,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:25,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:29,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:31,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:17:31,523 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:17:32,744 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:17:33,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:35,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:39,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:41,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:45,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:46,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:17:46,794 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:17:47,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:47,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:48,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:48,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:50,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:51,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:52,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:52,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:53,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:54,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:54,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:55,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:17:56,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:56,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:57,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:58,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:17:59,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:00,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:00,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:01,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:02,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:18:02,028 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:18:02,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:02,821 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:18:03,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:03,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:04,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:04,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:05,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:05,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:06,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:07,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:09,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:10,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:10,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:11,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:12,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:12,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:13,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:15,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:15,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:17,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:18:17,278 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:18:17,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:19,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:21,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:23,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:25,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:27,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:29,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:31,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:32,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:18:32,499 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:18:32,897 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:18:33,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:35,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:37,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:40,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:42,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:44,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:46,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:47,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:18:47,892 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:18:48,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:50,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:52,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:54,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:56,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:56,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:56,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:56,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:56,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:56,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:57,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:57,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:57,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:57,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:57,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:57,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:57,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:57,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:18:57,272 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 11:18:57,497 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 11:18:57,562 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 11:18:57,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:18:57,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 11:18:58,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:02,993 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:19:03,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:19:03,171 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:19:04,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:08,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:10,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:14,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:18,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:19:18,392 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:19:18,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:22,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:24,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:28,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:32,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:33,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:19:33,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:19:33,675 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:19:34,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:36,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:40,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:42,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:42,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:43,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:43,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:44,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:46,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:47,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:47,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:48,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:48,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:19:48,963 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:19:50,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:50,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:51,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:51,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:52,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:52,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:53,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:54,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:55,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:56,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:57,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:57,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:58,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:19:59,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:19:59,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:00,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:01,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:01,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:02,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:03,147 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:20:03,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:03,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:04,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:20:04,190 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:20:04,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:05,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:05,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:06,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:08,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:10,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:12,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:14,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:16,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:18,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:19,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:20:19,524 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:20:20,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:22,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:24,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:26,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:28,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:30,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:32,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:33,223 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:20:34,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:20:34,750 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:20:34,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:36,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:38,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:40,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:42,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:44,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:46,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:20:46,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:49,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:50,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:20:50,023 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:20:53,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:20:57,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:01,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:03,299 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:21:05,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:21:05,356 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:21:05,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:09,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:11,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:15,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:17,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:20,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:21:20,649 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:21:21,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:23,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:27,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:29,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:31,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:33,376 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:21:34,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:34,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:35,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:21:35,956 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:21:35,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:36,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:36,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:37,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:39,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:39,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:41,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:41,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:42,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:42,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:43,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:44,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:44,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:45,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:46,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:47,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:48,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:48,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:49,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:50,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:50,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:51,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:21:51,383 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:21:51,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:53,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:53,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:54,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:54,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:55,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:56,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:56,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:57,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:21:58,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:58,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:21:59,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:01,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:01,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:03,453 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:22:03,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:05,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:06,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:22:06,677 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:22:07,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:07,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:09,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:11,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:13,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:15,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:17,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:19,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:21,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:22:21,953 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:22:21,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:23,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:25,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:27,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:30,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:32,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:33,521 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:22:33,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 11:22:33,957 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 11:22:34,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:34,190 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 11:22:34,260 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 11:22:35,140 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:35,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 11:22:36,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:37,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:22:37,288 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:22:38,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:41,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:43,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:45,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:47,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:49,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:51,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:52,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:22:52,564 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:22:53,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:55,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:57,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:22:59,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:01,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:03,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:03,626 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:23:05,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:07,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:07,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:23:07,787 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:23:09,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:11,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:13,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:15,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:17,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:19,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:21,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:23,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:23:23,086 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:23:23,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:25,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:28,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:30,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:32,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:33,697 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:23:34,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:36,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:38,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:38,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:23:38,401 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:23:40,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:42,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:44,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:46,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:48,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:50,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:52,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:53,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:23:53,618 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:23:54,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:56,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:23:58,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:00,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:02,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:03,767 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:24:04,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:06,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:08,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:08,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:24:08,930 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:24:10,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:12,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:14,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:16,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:18,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:20,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:22,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:24,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:24,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:24:24,198 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:24:26,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:28,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:30,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:32,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:34,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:35,586 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:24:36,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:38,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:40,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:42,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:44,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:44,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:24:44,246 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:24:46,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:48,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:50,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:52,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:54,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:56,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:58,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:24:59,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:24:59,472 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:25:00,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:03,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:03,916 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:25:05,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:07,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:09,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:11,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:13,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:14,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:25:14,854 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:25:15,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:17,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:19,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:21,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:23,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:25,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:27,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:29,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:30,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:25:30,072 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:25:31,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:33,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:33,992 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:25:35,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:37,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:39,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:41,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:43,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:45,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:45,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:25:45,382 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:25:47,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:49,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:51,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:53,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:55,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:57,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:25:59,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:00,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:26:00,610 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:26:01,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:03,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:04,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:26:05,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:07,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:09,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:11,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:13,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:15,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:15,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:26:15,924 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:26:18,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:20,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:22,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:24,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:26,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:28,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:30,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:31,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:26:31,305 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:26:32,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:34,139 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:26:34,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:36,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:38,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:40,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:42,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:44,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:46,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:46,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:26:46,589 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:26:48,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:50,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:52,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:54,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:56,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:26:58,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:00,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:01,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:27:01,824 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:27:02,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:04,215 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:27:04,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:07,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:09,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:11,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:13,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:15,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:17,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:27:17,197 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:27:17,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:19,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:21,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:23,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:25,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:27,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:29,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:31,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:32,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:27:32,464 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:27:33,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:34,289 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:27:35,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:37,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:39,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:41,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:43,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:45,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:47,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:47,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:27:47,701 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:27:49,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:51,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:53,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:55,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:57,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:27:59,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:01,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:02,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:28:02,916 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:28:03,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:04,350 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:28:05,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:07,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:09,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:11,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:13,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:15,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:17,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:18,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:28:18,201 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:28:19,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:21,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:23,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:25,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:27,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:29,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:31,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:33,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:33,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:28:33,510 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:28:34,419 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:28:35,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:37,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:39,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:41,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:44,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:46,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:48,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:48,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:28:48,765 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:28:50,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:52,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:54,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:56,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:28:58,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:00,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:02,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:03,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:29:03,988 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:29:04,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:04,494 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:29:06,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:08,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:10,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:12,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:14,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:16,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:18,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:19,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:29:19,249 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:29:20,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:22,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:24,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:26,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:28,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:30,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:32,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:34,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:34,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:29:34,564 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:29:34,789 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:29:36,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:38,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:40,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:42,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:44,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:46,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:48,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:49,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:29:49,822 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:29:50,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:52,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:54,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:57,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:29:59,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:01,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:03,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:04,644 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:30:05,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:30:05,081 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:30:05,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:07,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:09,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:11,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:13,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:15,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:17,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:19,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:20,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:30:20,491 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:30:21,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:23,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:25,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:27,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:29,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:31,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:33,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:34,719 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:30:35,355 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:35,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:30:35,805 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:30:37,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:39,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:41,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:43,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:45,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:47,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:49,361 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:51,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:30:51,028 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:30:51,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:53,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:55,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:57,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:30:59,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:01,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:03,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:04,794 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:31:05,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:06,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:31:06,393 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:31:07,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:09,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:11,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:13,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:15,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:17,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:19,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:21,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:21,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:31:21,627 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:31:24,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:26,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:28,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:30,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:32,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:34,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:34,869 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:31:36,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:36,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:31:36,847 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:31:38,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:40,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:42,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:44,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:46,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:48,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:50,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:52,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:31:52,236 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:31:52,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:54,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:56,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:31:58,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:00,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:02,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:04,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:04,943 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:32:06,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:07,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:32:07,527 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:32:08,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:10,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:12,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:14,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:16,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:18,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:20,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:22,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:22,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:32:22,781 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:32:24,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:26,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:28,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:30,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:32,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:34,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:35,017 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:32:36,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:38,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:32:38,007 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:32:38,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:40,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:42,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:44,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:46,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:48,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:50,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:52,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:53,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:32:53,396 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:32:54,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:56,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:32:58,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:00,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:02,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:05,094 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:33:05,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:07,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:08,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:33:08,680 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:33:09,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:11,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:13,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:15,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:17,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:19,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:21,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:23,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:23,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:33:23,910 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:33:25,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:27,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:29,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:31,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:33,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:35,166 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:33:35,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:37,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:39,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:33:39,126 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:33:39,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:41,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:43,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:45,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:47,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:49,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:51,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:53,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:54,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:33:54,408 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:33:55,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:57,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:33:59,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:01,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:03,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:05,236 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:34:05,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:07,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:09,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:09,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:34:09,698 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:34:11,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:13,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:15,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:17,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:19,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:21,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:23,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:24,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:34:24,965 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:34:25,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:27,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:29,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:31,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:33,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:35,311 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:34:35,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:37,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:39,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:40,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:34:40,297 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:34:41,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:43,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:46,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:48,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:50,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:52,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:54,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:55,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:34:55,520 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:34:56,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:34:58,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:00,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:02,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:04,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:05,385 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:35:06,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:08,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:10,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:10,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:35:10,825 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:35:12,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:14,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:16,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:18,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:20,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:22,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:24,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:26,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:35:26,055 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:35:26,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:28,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:30,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:32,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:34,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:35,459 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:35:36,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:38,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:40,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:41,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:35:41,376 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:35:42,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:44,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:46,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:48,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:50,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:52,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:54,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:56,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:35:56,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:35:56,670 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:35:59,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:01,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:03,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:05,506 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:05,540 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:36:07,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:09,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:11,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:12,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:36:12,039 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:36:13,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:15,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:17,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:19,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:21,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:23,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:25,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:27,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:36:27,261 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:36:27,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:29,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:31,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:33,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:35,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:35,618 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:36:37,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:39,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:41,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:42,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:36:42,482 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:36:43,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:45,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:47,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:49,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:51,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:53,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:55,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:57,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:36:57,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:36:57,881 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:36:59,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:01,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:03,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:05,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:05,694 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:37:07,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:09,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:11,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:13,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:37:13,176 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:37:13,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:15,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:17,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:19,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:21,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:23,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:25,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:27,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:28,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:37:28,439 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:37:29,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:31,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:33,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:35,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:35,766 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:37:37,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:39,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:42,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:43,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:37:43,723 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:37:44,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:46,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:48,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:50,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:52,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:54,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:56,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:58,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:37:58,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:37:58,940 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:38:00,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:02,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:04,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:05,841 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:38:06,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:08,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:10,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:12,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:14,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:38:14,179 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:38:14,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:16,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:18,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:20,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:22,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:24,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:26,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:28,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:29,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:38:29,464 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:38:30,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:32,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:34,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:35,915 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:38:36,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:38,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:40,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:42,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:44,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:44,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:38:44,745 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:38:47,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:49,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:51,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:53,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:55,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:57,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:59,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:38:59,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:38:59,970 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:39:01,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:03,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:05,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:05,992 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:39:07,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:09,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:11,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:13,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:15,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:39:15,243 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:39:15,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:17,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:19,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:21,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:23,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:25,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:27,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:29,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:30,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:39:30,477 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:39:31,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:33,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:35,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:36,069 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:39:37,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:39,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:41,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:43,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:45,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:39:45,766 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:39:46,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:48,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:50,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:52,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:54,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:56,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:39:58,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:00,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:01,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:40:01,006 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:40:02,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:04,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:06,147 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:40:06,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:08,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:10,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:12,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:14,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:16,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:40:16,441 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:40:16,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:18,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:20,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:22,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:24,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:26,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:28,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:30,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:31,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:40:31,785 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:40:32,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:34,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:36,225 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:40:36,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:38,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:40,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:42,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:44,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:46,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:47,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:40:47,013 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:40:48,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:50,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:52,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:54,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:56,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:40:58,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:00,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:02,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:41:02,269 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:41:03,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:05,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:06,303 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:41:07,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:09,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:11,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:13,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:15,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:17,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:41:17,574 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:41:17,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:19,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:21,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:23,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:25,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:27,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:29,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:31,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:32,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:41:32,802 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:41:33,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:35,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:36,379 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:41:37,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:39,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:41,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:43,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:45,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:47,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:48,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:41:48,117 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:41:49,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:51,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:53,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:55,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:57,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:41:59,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:01,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:03,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:42:03,344 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:42:03,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:05,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:06,454 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:42:07,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:09,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:11,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:13,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:15,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:17,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:18,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:42:18,617 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:42:19,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:21,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:23,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:25,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:27,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:29,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:31,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:33,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:33,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:42:33,897 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:42:35,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:36,531 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:42:37,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:40,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:42,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:44,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:46,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:48,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:49,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:42:49,130 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:42:50,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:52,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:54,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:56,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:42:58,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:00,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:02,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:04,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:43:04,353 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:43:04,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:06,605 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:43:06,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:08,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:10,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:12,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:14,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:16,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:18,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:19,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:43:19,682 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:43:20,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:22,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:24,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:26,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:28,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:30,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:32,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:34,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:34,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:43:34,902 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:43:36,681 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:43:36,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:38,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:40,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:42,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:45,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:47,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:49,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:50,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:43:50,190 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:43:51,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:53,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:55,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:57,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:43:59,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:01,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:03,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:05,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:44:05,414 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:44:05,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:06,758 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:44:07,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:09,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:11,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:13,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:15,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:17,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:19,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:20,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:44:20,769 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:44:21,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:23,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:25,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:27,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:29,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:31,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:33,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:35,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:36,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:44:36,054 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:44:36,832 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:44:37,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:39,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:41,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:43,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:45,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:47,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:49,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:51,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:44:51,367 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:44:51,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:54,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:56,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:44:58,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:00,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:02,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:04,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:06,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:45:06,607 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:45:06,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:06,911 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:45:08,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:10,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:12,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:14,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:16,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:18,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:20,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:21,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:45:21,834 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:45:22,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:24,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:26,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:28,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:30,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:32,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:34,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:36,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:36,983 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:45:37,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:45:37,064 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:45:38,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:40,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:42,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:44,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:46,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:48,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:50,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:52,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:45:52,334 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:45:52,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:54,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:56,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:45:58,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:01,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:03,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:05,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:07,058 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:46:07,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:46:07,556 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:46:07,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:09,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:11,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:13,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:15,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:17,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:19,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:21,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:22,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:46:22,952 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:46:23,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:25,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:27,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:29,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:31,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:33,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:35,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:37,128 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:46:37,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:38,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:46:38,175 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:46:39,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:41,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:43,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:45,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:47,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:49,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:51,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:53,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:46:53,452 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:46:53,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:55,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:57,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:46:59,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:01,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:03,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:05,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:07,202 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:47:07,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:08,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:47:08,674 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:47:09,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:11,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:13,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:15,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:17,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:20,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:22,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:23,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:47:23,979 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:47:24,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:26,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:28,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:30,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:32,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:34,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:36,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:37,275 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:47:38,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:39,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:47:39,306 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:47:40,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:42,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:44,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:46,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:48,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:50,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:52,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:54,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:47:54,522 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:47:54,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:56,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:47:58,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:00,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:02,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:04,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:06,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:07,338 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:48:08,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:09,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:48:09,794 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:48:10,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:12,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:14,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:16,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:18,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:20,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:22,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:24,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:25,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:48:25,055 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:48:26,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:28,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:30,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:32,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:35,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:37,408 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:48:37,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:39,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:40,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:48:40,334 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:48:41,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:43,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:45,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:47,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:49,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:51,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:53,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:55,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:48:55,563 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:48:55,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:57,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:48:59,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:01,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:03,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:05,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:07,481 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:49:07,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:09,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:10,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:49:10,899 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:49:11,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:13,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:15,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:17,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:19,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:21,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:23,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:25,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:26,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:49:26,122 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:49:27,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:29,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:31,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:33,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:35,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:37,556 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:49:37,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:39,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:41,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:49:41,404 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:49:42,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:44,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:46,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:48,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:50,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:52,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:54,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:56,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:49:56,651 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:49:56,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:49:58,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:00,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:02,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:04,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:06,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:07,632 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:50:08,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:10,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:11,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:50:11,932 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:50:12,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:14,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:16,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:18,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:20,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:22,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:24,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:26,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:27,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:50:27,150 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:50:28,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:30,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:32,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:34,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:36,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:37,705 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:50:38,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:40,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:42,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:50:42,370 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:50:42,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:44,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:46,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:48,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:50,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:52,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:54,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:56,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:50:57,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:50:57,685 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:50:58,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:00,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:02,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:04,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:06,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:07,775 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:51:08,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:10,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:12,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:12,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:51:12,912 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:51:14,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:16,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:18,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:20,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:22,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:24,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:26,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:28,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:51:28,225 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:51:28,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:30,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:32,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:34,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:36,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:37,844 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:51:38,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:40,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:42,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:43,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:51:43,479 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:51:45,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:47,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:49,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:51,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:53,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:55,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:57,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:51:58,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:51:58,703 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:51:59,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:01,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:03,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:05,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:07,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:07,922 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:52:09,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:11,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:13,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:13,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:52:13,968 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:52:15,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:17,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:19,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:21,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:23,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:25,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:27,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:29,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:52:29,193 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:52:29,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:31,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:33,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:35,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:37,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:38,002 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:52:39,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:41,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:43,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:44,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:52:44,470 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:52:45,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:47,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:49,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:51,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:53,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:55,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:57,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:52:59,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:52:59,690 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:52:59,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:01,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:03,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:05,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:08,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:53:08,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:10,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:12,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:14,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:15,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:53:15,037 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:53:16,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:18,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:20,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:22,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:24,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:26,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:28,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:30,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:53:30,287 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:53:30,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:32,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:34,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:36,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:38,154 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:53:38,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:40,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:42,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:44,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:45,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:53:45,547 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:53:46,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:48,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:50,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:52,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:54,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:56,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:53:58,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:00,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:54:00,813 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:54:00,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:02,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:04,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:06,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:08,226 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:54:08,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:10,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:12,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:15,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:16,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:54:16,140 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:54:17,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:19,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:21,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:23,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:25,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:27,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:29,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:31,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:54:31,362 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:54:31,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:33,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:35,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:37,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:38,302 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:54:39,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:41,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:43,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:45,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:46,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:54:46,694 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:54:47,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:49,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:51,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:53,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:55,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:57,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:54:59,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:01,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:55:01,916 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:55:01,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:03,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:05,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:07,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:08,376 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:55:10,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:12,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:14,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:16,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:17,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:55:17,241 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:55:18,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:20,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:22,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:24,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:26,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:28,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:30,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:32,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:32,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:55:32,470 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:55:34,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:36,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:38,450 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:55:39,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:41,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:43,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:45,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:47,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:47,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:55:47,680 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:55:49,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:51,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:53,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:55,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:57,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:55:59,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:01,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:02,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:56:02,981 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:56:03,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:05,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:07,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:08,524 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:56:09,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:11,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:13,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:15,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:17,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:18,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:56:18,249 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:56:19,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:21,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:23,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:25,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:27,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:29,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:31,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:33,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:33,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:56:33,472 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:56:35,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:37,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:38,601 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:56:39,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:41,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:43,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:45,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:47,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:48,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:56:48,770 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:56:49,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:51,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:53,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:56,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:56:58,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:00,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:02,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:04,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:57:04,002 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:57:04,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:06,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:08,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:08,679 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:57:10,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:12,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:14,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:16,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:18,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:19,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:57:19,327 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:57:20,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:22,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:24,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:26,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:28,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:30,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:32,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:34,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:34,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:57:34,606 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:57:36,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:38,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:38,752 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:57:40,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:42,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:44,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:46,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:48,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:49,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:57:49,827 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:57:50,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:52,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:54,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:56,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:57:58,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:00,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:02,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:04,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:05,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:58:05,134 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:58:06,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:08,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:08,824 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:58:10,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:13,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:15,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:17,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:19,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:20,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:58:20,388 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:58:21,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:23,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:25,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:27,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:29,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:31,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:33,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:35,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:35,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:58:35,685 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:58:37,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:38,900 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:58:39,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:41,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:43,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:45,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:47,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:49,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:50,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:58:50,912 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:58:51,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:53,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:55,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:57,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:58:59,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:01,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:03,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:05,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:06,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:59:06,532 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:59:07,100 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:08,973 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:59:09,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:11,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:13,103 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:15,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:17,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:19,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:21,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:21,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:59:21,758 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:59:23,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:25,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:27,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:29,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:31,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:33,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:36,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:36,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:59:36,979 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:59:38,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:39,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 11:59:40,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:42,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:44,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:46,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:48,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:50,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:52,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:52,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 11:59:52,227 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 11:59:54,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:56,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 11:59:58,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:00,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:02,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:04,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:06,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:07,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:00:07,474 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:00:08,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:09,122 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:00:10,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:12,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:14,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:16,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:18,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:20,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:22,132 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:22,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:00:22,699 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:00:24,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:26,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:28,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:30,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:32,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:34,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:36,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:37,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:00:37,989 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:00:38,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:39,200 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:00:40,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:42,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:44,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:47,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:49,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:51,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:53,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:53,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:00:53,290 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:00:55,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:57,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:00:59,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:01,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:03,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:05,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:07,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:08,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:01:08,518 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:01:09,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:09,284 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:01:11,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:13,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:15,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:17,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:19,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:21,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:23,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:23,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:01:23,761 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:01:25,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:28,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:30,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:32,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:34,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:36,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:38,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:39,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:01:39,021 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:01:39,356 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:01:40,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:42,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:44,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:46,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:48,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:50,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:52,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:54,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:54,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:01:54,241 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:01:56,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:01:58,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:00,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:02,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:04,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:06,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:08,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:09,429 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:02:09,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:02:09,467 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:02:10,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:12,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:14,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:16,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:18,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:20,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:22,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:24,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:24,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:02:24,865 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:02:26,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:28,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:30,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:32,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:34,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:36,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:38,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:39,509 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:02:40,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:02:40,180 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:02:40,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:42,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:44,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:46,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:48,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:50,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:53,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:55,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:55,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:02:55,450 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:02:57,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:02:59,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:01,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:03,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:05,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:07,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:09,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:09,586 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:03:10,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:03:10,733 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:03:11,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:13,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:15,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:17,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:19,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:21,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:23,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:25,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:25,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:03:25,961 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:03:27,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:29,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:31,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:33,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:35,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:37,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:39,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:39,658 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:03:41,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:03:41,211 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:03:41,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:43,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:45,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:47,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:49,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:51,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:53,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:55,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:56,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:03:56,602 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:03:57,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:03:59,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:01,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:04,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:06,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:08,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:09,738 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:04:10,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:11,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:04:11,896 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:04:12,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:14,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:16,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:18,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:20,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:22,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:24,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:26,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:27,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:04:27,220 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:04:28,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:30,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:32,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:34,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:36,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:38,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:39,801 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:04:40,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:42,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:42,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:04:42,470 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:04:44,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:46,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:48,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:50,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:52,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:54,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:56,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:04:57,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:04:57,686 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:04:58,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:00,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:02,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:04,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:06,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:08,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:09,875 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:05:11,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:12,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:05:12,908 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:05:13,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:15,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:17,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:19,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:21,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:23,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:25,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:27,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:28,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:05:28,268 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:05:29,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:31,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:33,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:35,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:37,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:39,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:39,952 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:05:41,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:43,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:43,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:05:43,510 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:05:45,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:47,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:49,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:51,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:53,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:55,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:57,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:05:58,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:05:58,880 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:05:59,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:01,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:03,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:05,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:07,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:09,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:10,028 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:06:11,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:13,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:14,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:06:14,108 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:06:15,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:17,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:19,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:21,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:23,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:25,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:27,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:29,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:29,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:06:29,471 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:06:31,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:33,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:35,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:37,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:39,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:40,103 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:06:41,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:43,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:44,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:06:44,693 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:06:46,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:48,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:50,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:52,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:54,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:56,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:58,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:06:59,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:06:59,914 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:07:00,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:02,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:04,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:06,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:08,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:10,179 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:07:10,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:12,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:14,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:15,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:07:15,309 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:07:16,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:18,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:20,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:22,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:24,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:26,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:28,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:30,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:30,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:07:30,616 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:07:32,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:34,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:36,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:38,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:40,254 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:07:40,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:42,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:44,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:45,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:07:45,838 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:07:46,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:48,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:51,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:53,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:55,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:57,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:07:59,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:01,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:08:01,131 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:08:01,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:03,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:05,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:07,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:09,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:10,330 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:08:11,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:13,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:15,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:16,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:08:16,525 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:08:17,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:19,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:21,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:23,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:25,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:27,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:29,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:31,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:31,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:08:31,742 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:08:33,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:35,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:37,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:39,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:40,402 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:08:41,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:43,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:45,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:46,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:08:46,993 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:08:47,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:49,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:51,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:53,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:55,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:57,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:08:59,355 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:01,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:02,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:09:02,323 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:09:03,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:05,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:08,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:10,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:10,472 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:09:12,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:14,361 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:16,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:17,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:09:17,563 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:09:18,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:20,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:22,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:24,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:26,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:28,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:30,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:32,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:32,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:09:32,876 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:09:34,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:36,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:38,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:40,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:40,547 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:09:42,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:44,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:46,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:48,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:09:48,150 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:09:48,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:50,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:52,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:54,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:56,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:09:58,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:00,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:02,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:03,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:10:03,371 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:10:04,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:06,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:08,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:10,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:10,614 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:10:13,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:15,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:17,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:18,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:10:18,667 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:10:19,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:21,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:23,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:25,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:27,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:29,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:31,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:33,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:33,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:10:33,955 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:10:35,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:37,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:39,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:40,686 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:10:41,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:43,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:45,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:47,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:49,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:10:49,184 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:10:49,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:51,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:53,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:55,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:57,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:10:59,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:01,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:03,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:04,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:11:04,405 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:11:05,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:07,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:09,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:10,762 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:11:11,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:13,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:15,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:17,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:19,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:19,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:11:19,714 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:11:21,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:23,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:26,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:28,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:30,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:32,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:34,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:34,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:11:34,950 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:11:36,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:38,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:40,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:40,844 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:11:42,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:44,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:46,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:48,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:50,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:11:50,193 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:11:50,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:52,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:54,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:56,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:11:58,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:00,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:02,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:04,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:05,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:12:05,411 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:12:06,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:08,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:10,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:10,918 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:12:12,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:14,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:16,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:18,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:20,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:20,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:12:20,691 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:12:22,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:24,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:26,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:28,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:30,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:32,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:34,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:35,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:12:35,987 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:12:36,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:38,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:40,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:40,995 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:12:42,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:44,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:46,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:48,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:50,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:51,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:12:51,242 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:12:53,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:55,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:57,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:12:59,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:01,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:03,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:05,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:06,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:13:06,463 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:13:07,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:09,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:11,074 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:13:11,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:13,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:15,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:17,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:19,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:21,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:21,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:13:21,687 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:13:23,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:25,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:27,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:29,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:31,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:33,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:35,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:37,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:13:37,135 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:13:37,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:39,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:41,153 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:13:41,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:43,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:45,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:47,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:49,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:51,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:52,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:13:52,465 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:13:53,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:55,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:57,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:13:59,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:01,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:03,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:05,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:07,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:07,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:14:07,689 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:14:09,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:11,228 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:14:11,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:14,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:16,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:18,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:20,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:22,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:22,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:14:22,909 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:14:24,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:26,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:28,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:30,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:32,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:34,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:36,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:38,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:14:38,156 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:14:38,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:40,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:41,304 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:14:42,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:44,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:46,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:48,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:50,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:52,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:53,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:14:53,389 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:14:54,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:56,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:14:58,506 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:00,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:02,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:04,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:06,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:08,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:08,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:15:08,697 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:15:10,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:11,387 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:15:12,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:14,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:16,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:18,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:20,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:22,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:23,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:15:23,922 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:15:24,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:26,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:28,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:30,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:32,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:34,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:36,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:38,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:39,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:15:39,235 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:15:41,465 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:15:41,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:43,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:45,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:47,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:49,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:51,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:53,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:54,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:15:54,535 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:15:55,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:57,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:15:59,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:01,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:03,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:05,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:07,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:09,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:09,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:16:09,786 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:16:11,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:11,540 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:16:13,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:15,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:17,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:19,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:21,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:23,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:25,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:16:25,094 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:16:25,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:27,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:29,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:31,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:33,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:35,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:37,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:39,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:40,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:16:40,319 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:16:41,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:41,605 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:16:43,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:45,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:47,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:49,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:51,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:53,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:55,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:16:55,537 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:16:56,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:16:58,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:00,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:02,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:04,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:06,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:08,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:10,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:10,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:17:10,768 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:17:11,681 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:17:12,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:14,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:16,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:18,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:20,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:22,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:24,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:26,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:17:26,066 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:17:26,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:28,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:30,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:32,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:34,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:36,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:38,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:40,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:41,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:17:41,289 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:17:41,755 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:17:42,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:44,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:46,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:48,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:50,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:52,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:54,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:56,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:17:56,518 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:17:56,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:17:58,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:00,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:02,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:04,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:06,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:08,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:10,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:11,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:18:11,735 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:18:11,957 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:18:12,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:14,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:16,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:18,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:20,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:22,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:24,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:26,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:27,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:18:27,073 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:18:28,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:30,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:32,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:34,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:36,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:38,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:40,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:41,902 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:18:42,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:18:42,415 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:18:42,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:44,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:47,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:49,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:51,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:53,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:55,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:57,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:18:57,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:18:57,632 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:18:59,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:01,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:03,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:05,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:07,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:09,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:11,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:11,976 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:19:12,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:19:12,855 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:19:13,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:15,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:17,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:19,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:21,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:23,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:25,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:27,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:28,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:19:28,210 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:19:29,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:31,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:33,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:35,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:37,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:39,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:41,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:42,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:19:43,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:19:43,613 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:19:43,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:45,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:47,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:49,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:51,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:53,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:55,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:57,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:19:58,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:19:58,892 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:19:59,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:02,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:04,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:06,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:08,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:10,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:12,125 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:20:12,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:14,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:20:14,116 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:20:14,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:16,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:18,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:20,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:22,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:24,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:26,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:28,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:29,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:20:29,328 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:20:30,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:32,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:34,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:36,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:38,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:40,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:42,193 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:20:42,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:44,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:20:44,642 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:20:44,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:46,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:48,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:50,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:52,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:54,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:56,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:58,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:20:59,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:20:59,936 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:21:00,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:02,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:04,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:06,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:08,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:10,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:12,269 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:21:12,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:14,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:15,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:21:15,203 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:21:16,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:18,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:20,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:22,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:24,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:26,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:28,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:30,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:21:30,588 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:21:30,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:33,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:35,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:37,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:39,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:41,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:42,346 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:21:43,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:45,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:45,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:21:45,868 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:21:47,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:49,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:51,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:53,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:55,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:57,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:21:59,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:01,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:22:01,121 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:22:01,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:03,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:05,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:07,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:09,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:11,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:12,421 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:22:13,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:15,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:16,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:22:16,457 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:22:17,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:19,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:21,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:23,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:25,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:27,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:29,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:31,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:22:31,681 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:22:31,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:33,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:35,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:37,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:39,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:41,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:42,489 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:22:43,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:45,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:46,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:22:46,900 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:22:47,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:49,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:51,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:53,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:55,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:57,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:22:59,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:01,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:02,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:23:02,201 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:23:03,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:05,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:07,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:09,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:11,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:12,565 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:23:14,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:16,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:17,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:23:17,453 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:23:18,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:20,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:22,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:24,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:26,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:28,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:30,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:32,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:32,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:23:32,817 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:23:34,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:36,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:38,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:40,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:42,642 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:23:42,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:44,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:46,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:48,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:23:48,043 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:23:48,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:50,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:52,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:54,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:56,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:23:58,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:00,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:02,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:03,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:24:03,261 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:24:04,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:06,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:08,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:10,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:12,718 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:24:12,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:14,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:16,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:18,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:24:18,531 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:24:18,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:20,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:22,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:24,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:27,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:29,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:31,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:33,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:33,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:24:33,862 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:24:35,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:37,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:39,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:41,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:42,800 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:24:43,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:45,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:47,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:49,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:24:49,176 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:24:49,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:51,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:53,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:55,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:57,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:24:59,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:01,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:03,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:04,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:25:04,428 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:25:05,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:07,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:09,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:11,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:12,878 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:25:13,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:15,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:17,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:19,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:19,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:25:19,775 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:25:21,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:23,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:25,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:27,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:29,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:31,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:33,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:35,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:25:35,055 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:25:35,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:37,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:39,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:41,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:42,956 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:25:43,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:45,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:47,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:49,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:50,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:25:50,280 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:25:51,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:53,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:55,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:57,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:25:59,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:01,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:03,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:05,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:26:05,591 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:26:05,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:07,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:09,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:12,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:13,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:26:14,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:16,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:18,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:20,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:20,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:26:20,845 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:26:22,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:24,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:26,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:28,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:30,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:32,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:34,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:36,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:26:36,063 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:26:36,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:38,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:40,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:42,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:43,105 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:26:44,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:46,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:48,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:50,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:51,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:26:51,286 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:26:52,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:54,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:56,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:26:58,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:00,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:02,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:04,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:06,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:27:06,579 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:27:06,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:08,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:10,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:12,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:13,181 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:27:14,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:16,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:18,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:20,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:21,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:27:21,803 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:27:22,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:24,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:26,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:28,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:30,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:32,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:34,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:36,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:37,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:27:37,144 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:27:38,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:40,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:42,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:43,257 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:27:44,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:46,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:48,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:51,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:52,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:27:52,424 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:27:53,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:55,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:57,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:27:59,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:01,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:03,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:05,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:07,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:28:07,655 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:28:07,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:09,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:11,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:13,332 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:28:13,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:15,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:17,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:19,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:21,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:23,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:28:23,089 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:28:23,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:25,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:27,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:29,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:31,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:33,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:35,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:37,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:38,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:28:38,403 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:28:39,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:41,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:43,408 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:28:43,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:45,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:47,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:49,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:51,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:53,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:28:53,694 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:28:53,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:55,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:57,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:28:59,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:01,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:03,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:05,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:07,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:08,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:29:08,917 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:29:09,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:11,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:13,486 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:29:13,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:15,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:17,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:19,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:21,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:24,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:29:24,132 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:29:24,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:26,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:28,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:30,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:32,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:34,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:36,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:38,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:39,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:29:39,370 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:29:40,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:42,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:43,559 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:29:44,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:46,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:48,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:50,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:52,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:54,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:29:54,590 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:29:54,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:56,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:29:58,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:00,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:02,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:04,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:06,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:08,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:09,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:30:09,811 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:30:10,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:12,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:13,628 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:30:14,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:16,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:18,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:20,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:22,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:24,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:25,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:30:25,167 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:30:26,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:28,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:30,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:32,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:34,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:36,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:38,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:40,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:30:40,467 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:30:40,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:43,698 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:30:43,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:45,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:47,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:49,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:51,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:53,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:55,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:30:55,689 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:30:55,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:57,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:30:59,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:01,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:03,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:05,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:07,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:09,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:10,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:31:10,913 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:31:11,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:13,773 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:31:13,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:15,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:17,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:19,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:21,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:23,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:25,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:26,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:31:26,142 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:31:27,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:29,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:31,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:33,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:35,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:37,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:39,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:41,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:31:41,414 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:31:41,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:43,846 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:31:43,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:45,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:47,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:49,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:51,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:53,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:55,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:31:56,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:31:56,832 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:31:58,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:00,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:02,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:04,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:06,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:08,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:10,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:12,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:32:12,065 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:32:12,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:13,918 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:32:14,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:16,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:18,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:20,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:22,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:24,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:26,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:27,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:32:27,349 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:32:28,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:30,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:32,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:34,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:36,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:38,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:40,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:42,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:32:42,665 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:32:42,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:43,985 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:32:44,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:46,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:48,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:50,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:52,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:54,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:56,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:32:57,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:32:57,943 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:32:58,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:00,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:02,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:04,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:06,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:08,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:10,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:12,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:13,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:33:13,167 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:33:14,055 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:33:14,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:16,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:18,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:20,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:23,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:25,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:27,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:28,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:33:28,411 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:33:29,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:31,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:33,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:35,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:37,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:39,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:41,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:43,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:33:43,647 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:33:43,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:44,125 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:33:45,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:47,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:49,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:51,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:53,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:55,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:57,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:33:58,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:33:58,869 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:33:59,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:01,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:03,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:05,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:07,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:09,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:11,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:13,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:14,193 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:34:14,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:34:14,241 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:34:15,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:17,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:19,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:21,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:23,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:25,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:27,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:29,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:34:29,464 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:34:29,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:31,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:33,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:35,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:37,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:39,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:42,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:44,263 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:34:44,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:34:44,854 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:34:45,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:47,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:49,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:51,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:53,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:55,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:57,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:34:59,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:00,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:35:00,081 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:35:01,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:03,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:05,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:07,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:09,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:11,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:13,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:14,338 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:35:15,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:15,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:35:15,332 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:35:17,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:19,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:21,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:23,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:25,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:27,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:29,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:30,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:35:30,559 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:35:31,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:33,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:35,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:37,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:39,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:41,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:43,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:44,404 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:35:45,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:45,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:35:45,790 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:35:47,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:49,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:51,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:53,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:55,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:57,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:35:59,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:01,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:01,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:36:01,042 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:36:03,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:05,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:07,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:09,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:11,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:13,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:14,468 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:36:15,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:16,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:36:16,265 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:36:17,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:20,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:22,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:24,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:26,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:28,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:30,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:31,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:36:31,484 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:36:32,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:34,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:36,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:38,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:40,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:42,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:44,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:44,529 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:36:46,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:46,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:36:46,762 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:36:48,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:50,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:52,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:54,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:56,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:36:58,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:00,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:01,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:37:01,993 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:37:02,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:04,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:06,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:08,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:10,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:12,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:14,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:14,603 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:37:16,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:17,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:37:17,216 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:37:18,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:20,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:22,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:24,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:37:32,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:37:32,434 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:37:44,678 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:37:47,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:37:47,657 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:38:02,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:38:02,916 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:38:14,756 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:38:18,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:38:18,183 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:38:33,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:38:33,480 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:38:40,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:38:40,340 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 12:38:40,346 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 12:38:40,351 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 12:38:40,357 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 12:38:40,362 DEBUG SenderThread:3760704 [sender.py:send():232] send: metric -2022-08-30 12:38:40,368 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 12:38:40,397 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 12:38:40,475 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 12:38:41,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 12:38:42,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:38:43,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:38:44,838 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:38:46,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:38:48,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:38:48,718 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:38:58,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/config.yaml -2022-08-30 12:39:00,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:39:03,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:39:03,956 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:39:15,157 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:39:19,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:39:19,188 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:39:34,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:39:34,412 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:39:45,232 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:39:49,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:39:49,647 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:40:04,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:40:04,870 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:40:05,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:11,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:13,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:15,306 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:40:17,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:20,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:40:20,109 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:40:21,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:25,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:27,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:31,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:33,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:35,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:40:35,335 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:40:37,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:39,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:41,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:45,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:45,380 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:40:47,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:49,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:49,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:50,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:40:50,730 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:40:51,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:51,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:52,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:53,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:55,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:56,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:56,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:57,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:57,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:58,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:40:59,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:40:59,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:00,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:01,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:01,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:02,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:03,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:04,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:05,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:06,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:41:06,023 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:41:06,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:06,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:07,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:07,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:09,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:09,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:10,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:11,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:11,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:12,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:13,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:13,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:14,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:15,455 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:41:15,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:15,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:16,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:17,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:17,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:19,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:21,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:21,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:41:21,246 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:41:23,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:23,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:23,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:25,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:27,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:29,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:31,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:33,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:35,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:36,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:41:36,578 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:41:37,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:39,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:41,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:43,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:45,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:45,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:45,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:45,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:45,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:45,527 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:41:45,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:45,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:45,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:45,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:45,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:45,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:45,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:45,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:41:47,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:51,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:41:51,813 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:41:53,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:41:57,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:01,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:05,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:07,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:42:07,141 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:42:07,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:11,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:15,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:15,601 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:42:17,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:21,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:22,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:42:22,403 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:42:23,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:28,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:30,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:32,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:36,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:37,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:42:37,626 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:42:38,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:39,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:39,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:40,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:42,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:44,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:44,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:45,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:45,677 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:42:46,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:46,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:47,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:48,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:48,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:49,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:50,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:50,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:51,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:52,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:52,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:42:52,847 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:42:53,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:53,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:54,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:54,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:55,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:56,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:56,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:57,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:58,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:42:59,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:42:59,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:00,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:00,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:01,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:02,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:03,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:03,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:04,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:04,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:05,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:06,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:06,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:07,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:43:08,132 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:43:08,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:08,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:08,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:10,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:12,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:13,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:13,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:14,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:15,747 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:43:16,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:18,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:20,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:22,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:23,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:43:23,451 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:43:24,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:26,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:28,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:30,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:32,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:32,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:32,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:32,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:32,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:32,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:32,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:32,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:32,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:32,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:32,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:32,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:32,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:33,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:43:33,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 12:43:33,349 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 12:43:33,421 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 12:43:34,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:34,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 12:43:38,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:43:38,758 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:43:40,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:44,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:45,852 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:43:48,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:52,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:54,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:43:54,193 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:43:56,355 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:43:58,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:02,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:06,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:08,361 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:09,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:44:09,546 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:44:12,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:14,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:15,930 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:44:16,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:20,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:22,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:24,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:24,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:44:24,793 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:44:28,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:30,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:32,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:33,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:33,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:34,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:36,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:36,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:36,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:37,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:38,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:39,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:39,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:40,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:44:40,132 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:44:41,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:41,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:42,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:43,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:43,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:44,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:45,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:45,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,005 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:44:46,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:46,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:47,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:47,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:48,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:49,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:49,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:51,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:51,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:52,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:53,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:53,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:54,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:55,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:44:55,417 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:44:55,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:55,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:56,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:44:57,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:57,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:58,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:44:59,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:00,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:00,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:01,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:03,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:05,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:07,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:07,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:07,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:09,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:10,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:45:10,762 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:45:11,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:13,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:15,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:16,081 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:45:17,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:19,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:21,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:22,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:22,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:22,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:22,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:22,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:22,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:22,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:22,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:22,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:22,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:22,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:22,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:45:23,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:25,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:25,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:45:25,982 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:45:29,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:33,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:37,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:41,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:45:41,373 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:45:41,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:45,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:46,154 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:45:49,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:51,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:55,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:45:56,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:45:56,651 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:45:57,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:01,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:03,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:07,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:09,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:11,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:11,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:46:11,875 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:46:15,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:16,228 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:46:17,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:19,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:21,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:23,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:25,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:25,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:25,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:26,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:27,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:46:27,109 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:46:28,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:28,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:29,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:31,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:31,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:31,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:32,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:33,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:34,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:34,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:35,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:35,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:36,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:37,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:37,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:38,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:39,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:39,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:40,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:41,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:42,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:46:42,323 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:46:43,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:43,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:43,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:44,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:45,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:45,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,302 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:46:46,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:46,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:47,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:48,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:48,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:49,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:49,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:50,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:51,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:52,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:52,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:46:53,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:55,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:57,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:46:57,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:46:57,545 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:46:59,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:00,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:00,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:01,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:03,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:05,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:07,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:09,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:10,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:47:10,531 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 12:47:10,752 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 12:47:10,822 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 12:47:11,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:11,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 12:47:12,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:12,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:47:12,759 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:47:16,436 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:47:17,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:21,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:25,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:28,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:47:28,062 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:47:29,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:33,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:35,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:39,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:43,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:47:43,351 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:47:43,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:45,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:46,511 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:47:47,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:52,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:54,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:56,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:47:58,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:47:58,573 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:48:00,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:02,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:04,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:06,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:10,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:12,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:13,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:48:13,810 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:48:14,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:16,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:16,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:16,586 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:48:18,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:18,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:18,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:20,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:22,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:23,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:23,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:24,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:25,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:25,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:26,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:26,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:27,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:28,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:28,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:48:29,060 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:48:29,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:29,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:30,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:31,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:31,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:32,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:33,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:33,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:34,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:35,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:35,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:36,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:36,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:37,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:38,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:38,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:40,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:40,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:41,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:42,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:42,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:43,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:48:44,289 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:48:44,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:44,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:44,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:45,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:46,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:46,664 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:48:48,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:49,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:49,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:50,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:52,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:54,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:56,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:58,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:58,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:58,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:58,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:58,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:48:58,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:58,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:58,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:58,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:58,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:58,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:58,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:58,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:48:59,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:48:59,537 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:49:00,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:06,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:10,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:14,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:14,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:49:14,756 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:49:16,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:16,739 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:49:20,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:24,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:27,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:29,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:49:29,993 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:49:31,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:33,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:37,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:39,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:43,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:45,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:49:45,215 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:49:45,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:46,815 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:49:47,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:49,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:53,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:55,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:57,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:49:59,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:00,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:50:00,490 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:50:01,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:03,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:05,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:09,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:10,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:10,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:11,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:12,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:12,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:13,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:13,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:13,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:13,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:13,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:13,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:13,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:13,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:13,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:14,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:14,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:15,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:15,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:50:15,715 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:50:16,883 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:50:16,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:16,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:17,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:17,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:18,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:19,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:19,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:21,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:21,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:22,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:23,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:23,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:24,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:25,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:25,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:26,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:27,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:28,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:28,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:29,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:29,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:30,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:50:30,937 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:50:31,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:31,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:31,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:32,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:33,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:33,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:35,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:37,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:37,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:39,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:40,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:40,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:41,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:43,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:45,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:46,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:50:46,158 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:50:46,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:50:46,568 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 12:50:46,794 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 12:50:46,858 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 12:50:46,964 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:50:47,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:47,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 12:50:53,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:50:57,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:01,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:51:01,407 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:51:01,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:05,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:09,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:11,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:15,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:16,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:51:16,633 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:51:17,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:51:17,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:21,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:23,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:28,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:30,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:31,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:51:31,906 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:51:32,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:36,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:38,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:40,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:42,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:46,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:47,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:51:47,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:51:47,168 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:51:48,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:50,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:52,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:54,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:56,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:51:58,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:00,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:00,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:02,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:52:02,452 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:52:02,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:04,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:06,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:06,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:06,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:07,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:08,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:10,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:10,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:11,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:11,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:12,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:12,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:13,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:14,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:15,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:15,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:16,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:17,195 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:52:17,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:52:17,698 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:52:17,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:17,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:18,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:18,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:19,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:20,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:20,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:22,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:22,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:23,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:24,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:25,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:25,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:26,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:27,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:27,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:28,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:29,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:29,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:30,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:32,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:33,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:52:33,002 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:52:33,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:33,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:33,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:33,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:34,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:34,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:34,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:34,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:34,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:34,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:34,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:34,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:52:34,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:36,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:40,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:44,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:47,272 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:52:48,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:52:48,337 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:52:48,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:52,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:52:56,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:00,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:02,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:03,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:53:03,676 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:53:06,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:08,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:13,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:15,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:17,349 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:53:19,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:53:19,048 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:53:19,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:21,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:25,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:27,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:29,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:31,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:33,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:34,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:53:34,363 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:53:37,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:39,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:41,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:43,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:45,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:47,425 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:53:47,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:49,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:53:49,671 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:53:49,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:51,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:53,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:54,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:54,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:55,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:57,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:53:58,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:58,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:53:59,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:00,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:00,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:01,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:02,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:02,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:03,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:05,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:54:05,171 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:54:05,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:05,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:05,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:06,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:07,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:08,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:08,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:09,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:10,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:10,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:11,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:11,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:12,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:13,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:14,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:14,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:15,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:15,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:16,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,500 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:54:17,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:17,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:18,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:18,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:19,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:20,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:54:20,437 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:54:21,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:21,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:22,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:22,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:23,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:24,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:24,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:24,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:24,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:24,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:24,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:24,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:24,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:24,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:24,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:24,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:24,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:25,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:54:25,184 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 12:54:25,386 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 12:54:25,464 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 12:54:25,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:25,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 12:54:26,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:27,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:31,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:35,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:54:35,663 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:54:35,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:39,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:43,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:45,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:47,621 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:54:49,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:50,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:54:50,890 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:54:53,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:55,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:54:59,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:01,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:03,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:06,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:55:06,115 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:55:07,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:09,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:11,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:15,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:17,697 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:55:17,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:19,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:21,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:55:21,408 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:55:23,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:26,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:28,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:30,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:32,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:34,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:36,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:55:36,711 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:55:36,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:38,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:40,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:42,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:43,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:43,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:44,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:46,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:47,773 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:55:48,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:49,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:49,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:50,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:51,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:55:51,980 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:55:52,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:52,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:53,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:53,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:54,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:55,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:55,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:56,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:55:57,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:57,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:55:58,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:00,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:00,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:01,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:01,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:02,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:03,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:03,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:04,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:04,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:05,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:06,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:07,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:07,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:56:07,218 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:56:08,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:08,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:09,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:09,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:10,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:11,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:11,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:11,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:11,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:12,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:12,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:12,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:12,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:12,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:12,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:12,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:12,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:12,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:14,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:56:14,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:17,852 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:56:18,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:22,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:56:22,466 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:56:22,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:26,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:30,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:34,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:36,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:37,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:56:37,843 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:56:40,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:44,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:46,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:47,928 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:56:50,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:52,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:53,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:56:53,208 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:56:54,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:56:58,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:00,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:02,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:06,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:08,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:57:08,505 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:57:08,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:10,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:12,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:14,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:18,005 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:57:18,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:20,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:22,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:23,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:57:23,756 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:57:24,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:26,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:28,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:30,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:32,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:34,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:34,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:36,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:38,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:39,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:57:39,024 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:57:40,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:40,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:41,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:43,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:43,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:44,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:44,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:45,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:45,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:47,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:47,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,083 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:57:48,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:48,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:49,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:49,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:50,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:51,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:52,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:52,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:53,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:54,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:57:54,318 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:57:54,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:54,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:55,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:56,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:56,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:57:57,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:57,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:58,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:57:59,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:00,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:00,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:01,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 12:58:01,245 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 12:58:01,508 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 12:58:01,619 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 12:58:01,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:01,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 12:58:02,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:03,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:05,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:07,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:09,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:58:09,578 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:58:09,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:11,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:13,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:15,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:17,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:18,194 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:58:19,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:21,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:23,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:24,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:58:24,914 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:58:25,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:27,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:29,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:31,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:33,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:35,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:37,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:39,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:40,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:58:40,234 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:58:41,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:43,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:45,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:47,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:48,266 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:58:49,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:51,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:53,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:55,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:58:55,483 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:58:55,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:57,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:58:59,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:02,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:04,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:06,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:08,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:10,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:59:10,799 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:59:10,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:12,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:14,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:16,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:18,340 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:59:18,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:20,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:22,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:24,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:26,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:59:26,096 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:59:26,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:28,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:30,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:32,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:34,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:36,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:38,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:40,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:41,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:59:41,407 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:59:42,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:44,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:46,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:48,416 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 12:59:48,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:50,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:52,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:54,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:56,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 12:59:56,775 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 12:59:56,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 12:59:58,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:00,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:02,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:04,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:06,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:08,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:10,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:12,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:00:12,058 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:00:12,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:14,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:16,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:18,492 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:00:18,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:20,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:22,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:24,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:26,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:27,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:00:27,331 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:00:28,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:30,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:32,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:34,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:36,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:38,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:40,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:42,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:00:42,553 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:00:42,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:44,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:46,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:48,550 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:00:48,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:50,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:53,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:55,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:57,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:00:57,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:00:57,822 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:00:59,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:01,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:04,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:06,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:08,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:10,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:12,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:13,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:01:13,045 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:01:14,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:16,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:18,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:18,614 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:01:20,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:22,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:24,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:26,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:28,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:28,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:01:28,335 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:01:30,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:32,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:34,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:36,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:38,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:40,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:42,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:43,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:01:43,561 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:01:44,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:46,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:48,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:48,693 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:01:50,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:52,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:54,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:56,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:58,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:01:59,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:01:59,015 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:02:00,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:02,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:04,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:06,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:08,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:10,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:12,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:14,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:14,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:02:14,421 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:02:16,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:18,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:18,770 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:02:20,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:22,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:24,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:26,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:28,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:29,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:02:29,644 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:02:30,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:32,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:34,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:36,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:38,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:40,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:42,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:44,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:44,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:02:44,921 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:02:46,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:48,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:48,848 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:02:50,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:52,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:54,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:57,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:02:59,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:00,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:03:00,190 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:03:01,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:03,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:05,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:07,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:09,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:11,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:13,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:15,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:15,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:03:15,475 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:03:17,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:18,922 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:03:19,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:21,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:23,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:25,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:27,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:29,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:30,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:03:30,745 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:03:31,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:33,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:35,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:37,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:39,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:41,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:43,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:45,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:45,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:03:45,975 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:03:47,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:48,997 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:03:49,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:51,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:53,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:55,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:03:58,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:00,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:01,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:04:01,265 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:04:02,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:04,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:06,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:08,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:10,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:12,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:14,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:16,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:16,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:04:16,504 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:04:18,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:19,073 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:04:20,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:22,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:24,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:26,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:28,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:30,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:31,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:04:31,868 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:04:32,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:34,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:36,100 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:38,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:40,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:42,103 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:44,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:46,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:47,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:04:47,174 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:04:48,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:49,148 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:04:50,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:52,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:54,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:56,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:04:58,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:00,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:02,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:02,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:05:02,539 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:05:04,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:06,112 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:09,114 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:11,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:13,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:15,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:17,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:17,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:05:17,760 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:05:19,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:19,220 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:05:21,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:23,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:25,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:27,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:29,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:31,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:32,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:05:32,982 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:05:33,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:35,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:37,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:39,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:41,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:43,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:45,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:47,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:48,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:05:48,313 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:05:49,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:49,277 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:05:51,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:53,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:55,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:57,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:05:59,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:01,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:03,132 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:03,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:06:03,594 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:06:05,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:07,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:09,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:11,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:13,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:15,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:17,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:18,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:06:18,899 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:06:19,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:19,346 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:06:21,140 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:23,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:25,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:27,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:30,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:32,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:34,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:34,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:06:34,161 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:06:36,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:38,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:40,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:42,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:44,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:46,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:48,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:49,410 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:06:49,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:06:49,425 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:06:50,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:52,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:54,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:56,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:06:58,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:00,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:02,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:04,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:04,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:07:04,659 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:07:06,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:08,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:10,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:12,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:14,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:16,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:18,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:19,481 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:07:19,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:07:19,911 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:07:20,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:22,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:24,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:26,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:28,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:30,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:32,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:34,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:35,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:07:35,134 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:07:36,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:38,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:40,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:42,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:44,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:46,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:48,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:49,558 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:07:50,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:50,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:07:50,534 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:07:52,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:54,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:56,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:07:59,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:01,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:03,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:05,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:05,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:08:05,795 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:08:07,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:09,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:11,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:13,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:15,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:17,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:19,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:19,633 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:08:21,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:08:21,076 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:08:21,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:23,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:25,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:27,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:29,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:31,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:33,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:35,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:36,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:08:36,437 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:08:37,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:39,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:41,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:43,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:45,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:47,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:49,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:49,707 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:08:51,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:51,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:08:51,658 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:08:53,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:55,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:57,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:08:59,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:01,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:03,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:05,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:06,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:09:06,967 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:09:07,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:09,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:11,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:13,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:15,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:17,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:19,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:19,782 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:09:21,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:22,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:09:22,270 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:09:23,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:25,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:27,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:29,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:31,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:33,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:35,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:37,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:37,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:09:37,509 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:09:39,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:41,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:43,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:45,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:47,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:49,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:49,858 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:09:51,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:52,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:09:52,763 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:09:53,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:55,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:09:57,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:00,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:02,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:04,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:06,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:08,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:10:08,051 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:10:08,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:10,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:12,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:14,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:16,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:18,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:19,939 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:10:20,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:22,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:23,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:10:23,276 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:10:24,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:26,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:28,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:30,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:32,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:34,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:36,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:38,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:38,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:10:38,614 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:10:40,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:42,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:44,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:46,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:48,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:50,012 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:10:50,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:52,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:54,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:10:54,047 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:10:54,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:56,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:10:58,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:00,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:02,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:04,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:06,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:08,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:09,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:11:09,268 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:11:10,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:12,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:14,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:16,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:19,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:20,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:11:21,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:23,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:24,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:11:24,794 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:11:25,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:27,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:29,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:31,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:33,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:35,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:37,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:39,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:40,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:11:40,036 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:11:41,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:43,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:45,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:47,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:49,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:50,166 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:11:51,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:53,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:55,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:55,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:11:55,395 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:11:57,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:11:59,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:01,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:03,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:05,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:07,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:09,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:10,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:12:10,641 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:12:11,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:13,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:15,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:17,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:19,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:20,226 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:12:21,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:23,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:25,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:26,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:12:26,120 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:12:27,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:29,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:31,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:33,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:35,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:37,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:39,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:41,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:41,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:12:41,501 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:12:43,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:45,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:47,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:49,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:50,299 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:12:52,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:54,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:56,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:12:56,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:12:56,857 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:12:58,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:00,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:02,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:04,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:06,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:08,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:10,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:12,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:13:12,087 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:13:12,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:14,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:16,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:18,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:20,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:20,371 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:13:22,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:24,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:26,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:27,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:13:27,593 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:13:28,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:30,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:32,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:34,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:36,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:38,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:40,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:42,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:42,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:13:42,815 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:13:44,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:46,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:48,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:50,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:50,452 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:13:52,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:54,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:56,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:13:58,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:13:58,168 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:13:58,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:00,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:02,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:05,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:07,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:09,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:11,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:13,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:13,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:14:13,446 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:14:15,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:17,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:19,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:20,532 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:14:21,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:23,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:25,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:27,355 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:28,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:14:28,718 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:14:29,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:31,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:33,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:35,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:37,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:39,361 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:41,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:43,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:43,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:14:43,980 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:14:45,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:47,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:49,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:50,606 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:14:51,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:54,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:56,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:58,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:14:59,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:14:59,229 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:15:00,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:02,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:04,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:06,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:08,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:10,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:12,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:14,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:14,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:15:14,564 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:15:16,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:18,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:20,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:20,682 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:15:22,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:24,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:26,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:28,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:29,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:15:29,785 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:15:30,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:32,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:34,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:36,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:38,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:41,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:43,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:45,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:15:45,137 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:15:45,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:47,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:49,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:50,756 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:15:51,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:53,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:55,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:57,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:15:59,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:00,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:16:00,423 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:16:01,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:03,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:05,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:07,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:09,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:11,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:13,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:15,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:15,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:16:15,724 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:16:17,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:19,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:20,833 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:16:21,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:23,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:25,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:27,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:29,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:30,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:16:30,962 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:16:31,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:33,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:35,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:37,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:39,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:41,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:43,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:45,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:46,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:16:46,199 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:16:47,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:49,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:50,911 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:16:51,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:53,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:55,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:16:58,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:00,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:01,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:17:01,522 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:17:02,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:04,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:06,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:08,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:10,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:12,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:14,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:16,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:16,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:17:16,906 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:17:18,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:20,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:20,989 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:17:22,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:24,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:26,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:28,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:30,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:32,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:17:32,301 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:17:32,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:34,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:36,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:38,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:40,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:42,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:44,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:46,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:47,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:17:47,564 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:17:48,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:50,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:51,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:17:52,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:54,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:56,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:17:58,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:00,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:02,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:02,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:18:02,870 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:18:04,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:06,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:09,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:11,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:13,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:15,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:17,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:18,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:18:18,205 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:18:19,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:21,140 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:18:21,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:23,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:25,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:27,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:29,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:31,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:33,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:33,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:18:33,564 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:18:35,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:37,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:39,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:41,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:43,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:45,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:47,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:48,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:18:48,869 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:18:49,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:51,216 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:18:51,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:53,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:55,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:57,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:18:59,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:01,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:03,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:04,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:19:04,140 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:19:05,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:07,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:09,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:11,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:13,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:15,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:17,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:19,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:19:19,475 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:19:19,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:21,296 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:19:21,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:23,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:25,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:27,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:29,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:31,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:33,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:34,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:19:34,833 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:19:35,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:37,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:39,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:41,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:43,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:45,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:47,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:49,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:50,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:19:50,120 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:19:51,373 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:19:51,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:53,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:56,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:19:58,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:00,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:02,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:04,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:05,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:20:05,592 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:20:06,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:08,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:10,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:12,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:14,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:16,506 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:18,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:20,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:21,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:20:21,196 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:20:21,525 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:20:22,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:24,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:27,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:29,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:31,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:33,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:35,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:36,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:20:36,526 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:20:37,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:39,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:41,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:43,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:45,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:47,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:49,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:51,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:51,525 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:20:51,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:20:51,852 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:20:53,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:55,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:57,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:20:59,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:01,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:03,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:05,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:07,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:21:07,229 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:21:07,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:09,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:11,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:13,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:15,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:17,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:19,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:21,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:21,595 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:21:22,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:21:22,460 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:21:23,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:25,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:27,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:29,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:31,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:33,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:35,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:37,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:37,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:21:37,680 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:21:39,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:41,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:43,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:45,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:47,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:49,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:51,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:51,670 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:21:53,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:21:53,056 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:21:53,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:55,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:57,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:21:59,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:01,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:03,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:05,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:07,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:08,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:22:08,276 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:22:09,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:11,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:14,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:16,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:18,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:20,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:21,746 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:22:22,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:23,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:22:23,543 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:22:24,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:26,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:28,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:30,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:32,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:34,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:36,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:38,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:38,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:22:38,879 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:22:40,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:42,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:44,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:46,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:48,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:50,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:51,833 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:22:52,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:54,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:22:54,140 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:22:54,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:56,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:22:58,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:00,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:02,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:04,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:06,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:08,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:09,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:23:09,464 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:23:10,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:12,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:14,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:16,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:18,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:20,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:21,910 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:23:22,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:24,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:24,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:23:24,774 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:23:26,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:28,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:30,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:32,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:34,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:36,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:39,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:40,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:23:40,076 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:23:41,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:43,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:45,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:47,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:49,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:51,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:51,986 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:23:53,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:55,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:23:55,354 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:23:55,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:57,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:23:59,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:01,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:03,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:05,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:07,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:09,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:10,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:24:10,670 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:24:11,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:13,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:15,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:17,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:19,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:21,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:22,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:24:23,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:25,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:25,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:24:25,920 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:24:27,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:29,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:31,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:33,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:35,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:37,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:39,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:41,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:24:41,233 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:24:41,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:43,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:45,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:48,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:50,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:52,143 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:24:52,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:54,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:56,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:24:56,504 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:24:56,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:24:58,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:00,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:02,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:04,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:06,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:08,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:10,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:11,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:25:11,884 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:25:12,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:14,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:16,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:18,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:20,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:22,213 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:25:22,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:24,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:26,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:27,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:25:27,122 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:25:28,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:30,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:32,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:34,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:36,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:38,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:40,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:42,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:25:42,352 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:25:42,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:44,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:46,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:48,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:50,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:52,278 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:25:52,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:54,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:56,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:25:57,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:25:57,612 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:25:58,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:00,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:02,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:04,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:06,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:08,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:10,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:12,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:12,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:26:12,834 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:26:14,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:17,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:19,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:21,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:22,352 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:26:23,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:25,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:27,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:28,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:26:28,333 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:26:29,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:31,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:33,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:35,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:37,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:39,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:41,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:43,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:26:43,674 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:26:43,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:45,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:47,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:49,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:51,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:52,431 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:26:53,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:55,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:57,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:26:58,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:26:58,962 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:26:59,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:01,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:03,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:05,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:07,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:09,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:11,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:13,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:14,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:27:14,188 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:27:15,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:17,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:19,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:21,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:22,508 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:27:23,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:25,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:27,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:29,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:27:29,603 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:27:29,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:31,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:33,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:36,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:38,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:40,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:42,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:44,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:44,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:27:44,934 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:27:46,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:48,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:50,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:52,584 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:27:52,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:54,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:56,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:27:58,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:00,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:28:00,159 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:28:00,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:02,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:04,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:06,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:08,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:10,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:12,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:14,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:15,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:28:15,489 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:28:16,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:18,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:20,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:22,668 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:28:22,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:24,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:26,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:28,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:30,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:28:30,719 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:28:30,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:32,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:34,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:36,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:39,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:41,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:43,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:45,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:45,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:28:45,995 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:28:47,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:49,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:51,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:52,742 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:28:53,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:55,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:57,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:28:59,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:01,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:29:01,270 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:29:01,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:03,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:05,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:07,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:09,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:11,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:13,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:15,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:16,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:29:16,528 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:29:17,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:19,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:21,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:22,816 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:29:23,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:25,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:27,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:29,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:31,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:29:31,751 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:29:31,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:33,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:35,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:37,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:39,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:41,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:43,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:45,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:47,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:29:47,117 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:29:47,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:49,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:51,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:52,891 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:29:53,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:55,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:57,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:29:59,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:01,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:02,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:30:02,342 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:30:03,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:06,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:08,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:10,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:12,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:14,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:16,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:17,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:30:17,563 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:30:18,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:20,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:22,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:22,970 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:30:24,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:26,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:28,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:30,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:32,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:32,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:30:32,929 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:30:34,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:36,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:38,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:40,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:42,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:44,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:46,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:48,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:30:48,195 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:30:48,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:50,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:52,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:53,049 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:30:54,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:56,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:30:58,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:00,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:03,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:31:03,485 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:31:03,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:05,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:07,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:09,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:11,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:13,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:15,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:17,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:18,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:31:18,806 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:31:19,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:21,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:23,125 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:31:23,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:25,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:27,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:29,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:31,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:33,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:34,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:31:34,061 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:31:35,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:37,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:39,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:41,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:43,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:45,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:47,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:49,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:31:49,286 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:31:49,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:51,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:53,201 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:31:53,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:55,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:57,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:31:59,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:01,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:03,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:04,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:32:04,548 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:32:05,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:07,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:09,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:11,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:13,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:15,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:17,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:19,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:19,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:32:19,944 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:32:21,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:23,277 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:32:23,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:25,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:27,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:29,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:31,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:33,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:35,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:32:35,165 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:32:35,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:37,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:39,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:41,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:43,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:45,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:47,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:49,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:50,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:32:50,495 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:32:51,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:53,353 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:32:53,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:55,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:57,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:32:59,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:01,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:03,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:05,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:33:05,717 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:33:05,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:07,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:10,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:12,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:14,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:16,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:18,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:20,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:20,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:33:20,969 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:33:22,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:23,428 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:33:24,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:26,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:28,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:30,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:32,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:34,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:36,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:33:36,341 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:33:36,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:38,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:40,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:42,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:44,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:46,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:48,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:50,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:51,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:33:51,640 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:33:52,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:53,504 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:33:54,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:56,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:33:58,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:00,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:02,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:04,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:06,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:06,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:34:06,893 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:34:08,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:10,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:12,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:14,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:16,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:18,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:21,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:22,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:34:22,190 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:34:23,580 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:34:23,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:25,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:27,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:29,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:31,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:33,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:35,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:37,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:34:37,415 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:34:37,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:39,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:41,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:43,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:45,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:47,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:49,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:51,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:52,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:34:52,659 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:34:53,657 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:34:53,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:55,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:57,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:34:59,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:01,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:03,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:05,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:07,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:07,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:35:07,969 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:35:09,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:11,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:13,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:16,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:18,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:20,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:22,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:23,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:35:23,294 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:35:23,733 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:35:24,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:26,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:28,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:30,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:32,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:34,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:36,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:38,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:35:38,567 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:35:38,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:40,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:42,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:44,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:46,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:48,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:50,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:52,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:53,812 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:35:53,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:35:53,869 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:35:54,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:56,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:35:58,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:00,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:02,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:04,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:06,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:08,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:09,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:36:09,191 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:36:10,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:12,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:14,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:16,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:18,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:20,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:22,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:23,889 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:36:24,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:36:24,512 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:36:24,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:26,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:28,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:30,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:32,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:34,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:36,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:38,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:39,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:36:39,789 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:36:40,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:42,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:44,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:46,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:48,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:50,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:52,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:53,965 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:36:54,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:55,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:36:55,063 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:36:56,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:36:58,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:01,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:03,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:05,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:07,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:09,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:10,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:37:10,438 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:37:11,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:13,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:15,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:17,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:19,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:21,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:23,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:24,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:37:25,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:37:25,664 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:37:25,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:27,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:29,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:31,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:33,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:35,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:37,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:39,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:40,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:37:40,931 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:37:41,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:43,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:45,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:47,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:49,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:51,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:53,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:54,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:37:55,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:56,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:37:56,306 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:37:57,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:37:59,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:01,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:03,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:05,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:07,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:09,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:11,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:38:11,529 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:38:11,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:13,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:15,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:17,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:19,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:21,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:24,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:24,164 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:38:26,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:26,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:38:26,772 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:38:28,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:30,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:32,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:34,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:36,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:38,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:40,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:42,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:42,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:38:42,077 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:38:44,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:46,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:48,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:50,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:52,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:54,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:54,236 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:38:56,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:38:57,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:38:57,359 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:38:58,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:00,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:03,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:05,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:07,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:09,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:11,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:12,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:39:12,587 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:39:13,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:15,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:17,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:19,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:21,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:23,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:24,306 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:39:25,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:27,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:27,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:39:27,840 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:39:29,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:31,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:33,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:35,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:37,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:39,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:41,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:43,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:43,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:39:43,107 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:39:45,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:47,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:49,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:51,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:53,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:54,379 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:39:55,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:57,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:39:58,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:39:58,356 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:39:59,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:01,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:03,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:05,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:07,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:09,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:12,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:13,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:40:13,586 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:40:14,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:16,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:18,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:20,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:22,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:24,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:24,452 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:40:26,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:28,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:28,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:40:28,988 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:40:30,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:32,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:34,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:36,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:38,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:40,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:42,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:44,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:44,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:40:44,212 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:40:46,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:48,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:50,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:52,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:54,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:54,528 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:40:56,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:58,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:40:59,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:40:59,561 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:41:00,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:02,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:04,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:06,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:08,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:10,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:12,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:14,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:14,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:41:14,881 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:41:16,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:18,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:20,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:22,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:24,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:24,604 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:41:26,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:28,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:30,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:30,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:41:30,115 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:41:33,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:35,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:37,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:39,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:41,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:43,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:45,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:45,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:41:45,436 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:41:47,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:49,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:51,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:53,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:54,680 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:41:55,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:57,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:41:59,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:00,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:42:00,700 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:42:01,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:03,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:05,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:07,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:09,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:11,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:13,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:15,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:15,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:42:15,996 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:42:17,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:19,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:21,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:23,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:24,752 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:42:25,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:27,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:29,100 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:31,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:31,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:42:31,323 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:42:33,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:35,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:37,103 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:39,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:41,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:44,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:46,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:46,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:42:46,627 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:42:48,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:50,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:52,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:54,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:54,829 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:42:56,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:42:58,112 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:00,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:01,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:43:01,848 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:43:02,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:04,114 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:06,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:08,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:10,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:12,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:14,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:16,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:17,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:43:17,181 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:43:18,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:20,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:22,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:24,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:24,906 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:43:26,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:28,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:30,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:32,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:32,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:43:32,478 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:43:34,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:36,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:38,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:40,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:42,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:45,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:47,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:47,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:43:47,806 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:43:49,132 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:51,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:53,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:54,976 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:43:55,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:57,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:43:59,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:01,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:03,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:44:03,026 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:44:03,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:05,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:07,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:09,140 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:11,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:13,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:15,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:17,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:18,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:44:18,333 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:44:19,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:21,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:23,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:25,045 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:44:25,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:27,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:29,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:31,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:33,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:33,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:44:33,617 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:44:35,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:37,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:39,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:41,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:43,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:45,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:47,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:48,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:44:48,937 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:44:49,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:51,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:53,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:55,120 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:44:55,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:57,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:44:59,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:01,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:03,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:04,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:45:04,182 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:45:05,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:07,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:09,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:11,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:13,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:15,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:17,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:19,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:19,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:45:19,495 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:45:21,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:23,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:25,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:25,195 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:45:27,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:29,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:32,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:34,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:34,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:45:34,719 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:45:36,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:38,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:40,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:42,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:44,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:46,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:48,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:49,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:45:49,944 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:45:50,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:52,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:54,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:55,273 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:45:56,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:45:58,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:00,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:02,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:04,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:05,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:46:05,277 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:46:06,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:08,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:10,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:12,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:14,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:16,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:18,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:20,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:20,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:46:20,499 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:46:22,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:24,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:25,348 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:46:27,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:29,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:31,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:33,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:35,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:35,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:46:35,806 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:46:37,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:39,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:41,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:43,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:45,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:47,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:49,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:51,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:46:51,126 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:46:51,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:53,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:55,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:55,415 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:46:57,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:46:59,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:01,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:03,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:05,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:06,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:47:06,343 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:47:07,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:09,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:11,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:13,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:15,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:17,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:19,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:21,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:21,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:47:21,626 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:47:23,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:25,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:25,490 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:47:27,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:29,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:31,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:33,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:35,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:36,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:47:36,849 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:47:37,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:39,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:41,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:43,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:45,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:47,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:49,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:51,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:52,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:47:52,071 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:47:53,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:55,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:55,564 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:47:57,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:47:59,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:02,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:04,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:06,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:07,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:48:07,473 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:48:08,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:10,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:12,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:14,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:16,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:18,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:20,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:22,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:22,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:48:22,699 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:48:24,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:25,637 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:48:26,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:28,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:30,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:32,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:34,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:36,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:37,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:48:37,971 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:48:38,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:40,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:42,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:44,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:46,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:48,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:50,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:52,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:53,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:48:53,283 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:48:54,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:55,713 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:48:56,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:48:58,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:00,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:02,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:04,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:06,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:08,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:49:08,509 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:49:09,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:11,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:13,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:15,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:17,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:19,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:21,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:23,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:23,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:49:23,808 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:49:25,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:25,788 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:49:27,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:29,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:31,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:33,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:35,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:37,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:39,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:49:39,099 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:49:39,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:41,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:43,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:45,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:47,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:49,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:51,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:53,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:54,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:49:54,327 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:49:55,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:55,862 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:49:57,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:49:59,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:01,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:03,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:05,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:07,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:09,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:09,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:50:09,732 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:50:11,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:13,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:15,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:17,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:19,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:21,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:23,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:25,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:50:25,005 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:50:25,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:25,936 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:50:27,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:29,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:31,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:33,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:36,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:38,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:40,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:50:40,231 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:50:40,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:42,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:44,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:46,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:48,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:50,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:52,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:54,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:55,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:50:55,502 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:50:56,010 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:50:56,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:50:58,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:00,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:02,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:04,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:06,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:08,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:10,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:10,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:51:10,822 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:51:12,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:14,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:16,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:18,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:20,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:22,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:24,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:26,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:51:26,074 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:51:26,295 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:51:26,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:28,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:30,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:32,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:34,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:36,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:38,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:40,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:41,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:51:41,378 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:51:42,355 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:44,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:46,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:48,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:50,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:52,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:54,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:56,161 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:51:56,361 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:51:56,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:51:56,662 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:51:58,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:00,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:02,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:04,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:06,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:08,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:11,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:12,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:52:12,032 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:52:13,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:15,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:17,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:19,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:21,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:23,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:25,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:26,238 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:52:27,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:52:27,255 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:52:27,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:29,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:31,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:33,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:35,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:37,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:39,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:41,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:42,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:52:42,485 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:52:43,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:45,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:47,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:49,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:51,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:53,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:55,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:56,315 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:52:57,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:52:57,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:52:57,709 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:52:59,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:01,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:03,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:05,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:07,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:09,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:11,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:12,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:53:12,996 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:53:13,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:15,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:17,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:19,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:21,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:23,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:25,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:26,392 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:53:27,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:28,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:53:28,228 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:53:29,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:31,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:33,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:35,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:37,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:39,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:41,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:43,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:53:43,656 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:53:44,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:46,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:48,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:50,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:52,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:54,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:56,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:56,470 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:53:58,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:53:58,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:53:58,891 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:54:00,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:02,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:04,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:06,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:08,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:10,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:12,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:14,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:54:14,113 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:54:14,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:16,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:18,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:20,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:22,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:24,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:26,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:26,545 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:54:28,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:29,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:54:29,400 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:54:30,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:32,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:34,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:36,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:38,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:40,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:42,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:44,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:44,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:54:44,742 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:54:46,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:48,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:50,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:52,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:54,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:56,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:54:56,621 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:54:58,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:00,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:55:00,084 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:55:00,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:02,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:04,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:06,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:08,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:10,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:12,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:14,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:15,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:55:15,377 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:55:16,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:18,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:20,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:22,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:24,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:26,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:26,697 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:55:28,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:30,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:30,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:55:30,602 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:55:32,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:34,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:36,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:38,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:40,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:42,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:44,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:45,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:55:45,895 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:55:46,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:49,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:51,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:53,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:55,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:56,773 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:55:57,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:55:59,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:01,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:56:01,173 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:56:01,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:03,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:05,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:07,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:09,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:11,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:13,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:15,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:16,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:56:16,569 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:56:17,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:19,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:21,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:23,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:25,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:26,848 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:56:27,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:29,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:31,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:31,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:56:31,865 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:56:33,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:35,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:37,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:39,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:41,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:43,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:45,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:47,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:56:47,157 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:56:47,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:49,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:51,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:53,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:55,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:56,923 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:56:57,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:56:59,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:01,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:02,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:57:02,506 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:57:03,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:05,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:07,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:09,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:11,506 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:13,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:15,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:17,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:17,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:57:17,727 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:57:19,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:21,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:23,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:25,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:26,996 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:57:27,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:29,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:31,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:33,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:57:33,033 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:57:33,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:35,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:37,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:39,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:41,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:43,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:45,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:47,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:48,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:57:48,267 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:57:49,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:52,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:54,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:56,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:57:57,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:57:58,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:00,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:02,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:03,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:58:03,561 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:58:04,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:06,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:08,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:10,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:12,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:14,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:16,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:18,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:18,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:58:18,825 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:58:20,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:22,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:24,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:26,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:27,140 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:58:28,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:30,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:32,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:34,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:58:34,047 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:58:34,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:36,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:38,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:40,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:42,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:44,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:46,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:48,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:49,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:58:49,283 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:58:50,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:52,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:54,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:56,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:58:57,217 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:58:58,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:00,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:02,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:04,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:04,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:59:04,707 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:59:06,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:08,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:10,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:12,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:14,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:16,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:18,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:19,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:59:19,993 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:59:20,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:22,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:24,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:26,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:27,293 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:59:28,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:30,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:33,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:35,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:59:35,260 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:59:35,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:37,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:39,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:41,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:43,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:45,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:47,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:49,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:50,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 13:59:50,699 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 13:59:51,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:53,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:55,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:57,370 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 13:59:57,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 13:59:59,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:01,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:03,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:05,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:05,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:00:05,928 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:00:07,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:09,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:11,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:13,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:15,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:17,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:20,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:21,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:00:21,216 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:00:22,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:24,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:26,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:27,446 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:00:28,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:30,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:32,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:34,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:36,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:00:36,437 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:00:36,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:38,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:40,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:42,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:44,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:46,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:48,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:50,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:51,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:00:51,760 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:00:52,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:54,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:56,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:00:57,525 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:00:58,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:00,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:02,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:04,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:06,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:07,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:01:07,046 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:01:08,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:10,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:12,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:14,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:16,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:18,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:20,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:22,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:01:22,283 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:01:22,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:25,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:27,604 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:01:27,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:29,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:31,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:33,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:35,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:37,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:01:37,560 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:01:37,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:39,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:41,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:43,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:45,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:47,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:49,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:51,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:52,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:01:52,866 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:01:53,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:55,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:57,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:01:57,681 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:01:59,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:01,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:03,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:05,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:07,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:08,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:02:08,130 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:02:09,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:11,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:13,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:15,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:17,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:19,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:21,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:23,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:02:23,351 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:02:23,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:25,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:27,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:27,756 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:02:29,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:31,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:33,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:35,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:37,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:38,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:02:38,626 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:02:39,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:41,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:44,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:46,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:48,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:50,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:52,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:53,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:02:53,893 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:02:54,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:56,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:02:57,832 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:02:58,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:00,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:02,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:04,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:06,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:08,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:09,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:03:09,170 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:03:10,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:12,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:14,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:16,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:18,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:20,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:22,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:24,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:03:24,485 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:03:24,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:26,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:27,908 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:03:28,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:30,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:32,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:35,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:37,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:39,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:39,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:03:39,708 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:03:41,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:43,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:45,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:47,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:49,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:51,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:53,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:55,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:03:55,121 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:03:55,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:57,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:03:57,987 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:03:59,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:01,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:03,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:05,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:07,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:09,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:10,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:04:10,470 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:04:11,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:13,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:15,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:17,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:19,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:21,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:23,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:25,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:25,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:04:25,781 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:04:27,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:28,064 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:04:29,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:31,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:33,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:35,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:37,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:39,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:41,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:04:41,004 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:04:41,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:44,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:46,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:48,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:50,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:52,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:54,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:56,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:04:56,346 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:04:56,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:04:58,139 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:04:58,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:00,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:02,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:04,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:06,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:08,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:10,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:11,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:05:11,572 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:05:12,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:14,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:16,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:18,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:20,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:22,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:24,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:26,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:26,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:05:26,876 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:05:28,213 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:05:28,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:30,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:32,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:34,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:36,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:38,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:40,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:42,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:05:42,133 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:05:42,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:44,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:46,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:48,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:50,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:53,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:55,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:57,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:05:57,398 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:05:57,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:05:58,285 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:05:59,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:01,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:03,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:05,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:07,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:09,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:11,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:12,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:06:12,704 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:06:13,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:15,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:17,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:19,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:21,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:23,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:25,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:27,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:27,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:06:27,927 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:06:28,362 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:06:29,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:31,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:33,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:35,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:37,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:39,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:41,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:43,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:06:43,230 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:06:44,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:46,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:48,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:50,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:52,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:54,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:56,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:06:58,437 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:06:58,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:06:58,525 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:06:58,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:00,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:02,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:04,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:06,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:08,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:10,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:12,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:13,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:07:13,750 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:07:14,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:16,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:18,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:20,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:22,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:24,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:26,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:28,513 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:07:28,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:29,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:07:29,100 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:07:30,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:32,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:34,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:36,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:38,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:40,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:42,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:44,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:07:44,356 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:07:44,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:46,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:48,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:50,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:52,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:54,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:56,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:58,587 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:07:58,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:07:59,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:07:59,577 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:08:00,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:02,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:04,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:06,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:08,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:10,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:12,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:14,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:14,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:08:14,914 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:08:16,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:18,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:21,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:23,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:25,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:27,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:28,665 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:08:29,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:30,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:08:30,222 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:08:31,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:33,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:35,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:37,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:39,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:41,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:43,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:45,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:08:45,490 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:08:45,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:47,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:08:58,747 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:09:00,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:09:00,818 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:09:16,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:09:16,072 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:09:28,824 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:09:31,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:09:31,324 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:09:46,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:09:46,595 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:09:58,898 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:10:01,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:10:01,867 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:10:02,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:10:02,111 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 14:10:02,112 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 14:10:02,188 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 14:10:02,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:10:02,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 14:10:03,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:10:09,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:10:17,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:10:17,312 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:10:25,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:10:28,982 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:10:32,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:10:32,540 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:10:47,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:10:47,779 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:10:59,060 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:11:03,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:11:03,008 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:11:18,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:11:18,229 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:11:29,137 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:11:33,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:11:33,450 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:11:48,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:11:48,674 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:11:59,215 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:12:03,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:12:03,898 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:12:19,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:12:19,178 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:12:29,291 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:12:34,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:12:34,400 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:12:49,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:12:49,690 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:12:59,367 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:13:04,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:13:04,917 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:13:17,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:13:19,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:13:20,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:13:20,185 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:13:29,442 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:13:35,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:13:35,424 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:13:50,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:13:50,664 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:13:59,517 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:14:05,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:14:05,920 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:14:15,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:17,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:21,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:14:21,214 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:14:25,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:29,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:14:29,591 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:14:30,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:34,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:36,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:14:36,438 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:14:36,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:40,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:44,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:48,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:50,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:51,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:14:51,664 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:14:54,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:56,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:14:59,663 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:15:00,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:02,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:04,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:06,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:15:06,909 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:15:08,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:10,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:12,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:14,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:18,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:20,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:22,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:15:22,199 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:15:22,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:24,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:26,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:28,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:29,735 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:15:30,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:32,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:34,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:36,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:37,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:15:37,476 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:15:38,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:40,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:42,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:44,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:45,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:45,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:46,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:48,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:49,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:49,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:50,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:51,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:51,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:52,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:15:52,697 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:15:52,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:53,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:53,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:54,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:55,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:55,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:56,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:57,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:57,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:58,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:15:59,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:15:59,810 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:16:00,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:00,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:01,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:01,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:02,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:03,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:04,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:04,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:05,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:05,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:06,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:07,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:07,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:16:07,916 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:16:09,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:11,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:11,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:13,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:13,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:17,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:18,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:18,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:21,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:23,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:16:23,139 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:16:25,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:29,883 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:16:30,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:30,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:30,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:16:34,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:36,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:38,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:16:38,363 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:16:40,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:42,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:46,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:48,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:52,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:53,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:16:53,609 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:16:54,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:56,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:58,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:16:59,960 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:17:02,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:04,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:06,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:08,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:08,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:17:08,835 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:17:10,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:12,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:16,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:18,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:20,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:22,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:24,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:24,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:17:24,129 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:17:26,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:28,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:30,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:17:30,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:32,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:34,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:36,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:36,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:36,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:38,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:39,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:17:39,340 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:17:40,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:40,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:40,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:41,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:42,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:42,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:44,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:44,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:45,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:46,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:46,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:47,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:48,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:48,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:49,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:50,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:50,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:51,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:52,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:53,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:53,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:54,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:17:54,602 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:17:54,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:54,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:55,075 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 14:17:55,337 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 14:17:55,413 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 14:17:56,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:17:56,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 14:17:57,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:57,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:17:59,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:00,127 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:18:02,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:03,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:03,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:06,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:09,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:18:09,825 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:18:10,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:13,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:13,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:14,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:15,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:15,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:18,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:19,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:19,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:18:22,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:24,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:25,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:18:25,063 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:18:28,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:30,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:30,202 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:18:34,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:36,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:38,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:40,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:18:40,379 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:18:42,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:44,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:46,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:50,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:52,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:54,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:55,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:18:55,715 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:18:56,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:18:58,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:00,277 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:19:02,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:04,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:06,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:08,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:10,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:11,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:19:11,009 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:19:12,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:14,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:16,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:18,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:21,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:23,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:25,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:26,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:19:26,243 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:19:27,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:29,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:30,353 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:19:30,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:30,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:31,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:32,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:32,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:33,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:33,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:34,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:35,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:35,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:36,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:37,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:37,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:38,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:39,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:39,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:40,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:41,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:19:41,541 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:19:41,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:41,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:42,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:43,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:43,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:44,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:45,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:46,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:46,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:47,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:47,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:51,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:53,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:54,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:54,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:56,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:19:56,806 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:19:57,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:19:57,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:57,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:19:58,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:00,428 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:20:01,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:01,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:01,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:05,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:07,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:07,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:09,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:09,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:11,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:12,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:20:12,141 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:20:15,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:17,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:19,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:19,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:20:21,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:23,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:25,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:27,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:20:27,488 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:20:29,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:30,505 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:20:31,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:33,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:37,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:39,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:41,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:42,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:20:42,716 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:20:43,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:45,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:49,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:51,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:53,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:55,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:57,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:20:57,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:20:57,961 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:20:59,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:00,574 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:21:01,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:03,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:05,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:07,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:09,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:11,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:13,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:13,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:21:13,245 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:21:15,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:17,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:19,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:21,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:21,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:22,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:22,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:23,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:23,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:24,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:25,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:26,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:26,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:27,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:28,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:28,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:21:28,463 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:21:29,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:29,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:29,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,649 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:21:30,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:30,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:31,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:31,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:32,721 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 14:21:32,946 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 14:21:33,021 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 14:21:33,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:33,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 14:21:35,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:35,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:35,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:37,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:39,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:40,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:40,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:42,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:43,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:21:43,801 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:21:44,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:44,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:46,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:46,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:48,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:50,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:52,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:52,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:53,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:54,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:55,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:55,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:21:56,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:21:59,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:21:59,047 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:22:00,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:00,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:00,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:01,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:22:04,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:05,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:05,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:06,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:07,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:07,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:10,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:12,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:12,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:12,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:22:14,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:22:14,268 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:22:16,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:18,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:20,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:24,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:26,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:28,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:29,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:22:29,522 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:22:30,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:31,167 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:22:32,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:34,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:38,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:40,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:42,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:44,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:44,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:22:44,745 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:22:46,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:48,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:50,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:52,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:54,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:56,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:58,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:22:59,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:22:59,978 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:23:00,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:01,242 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:23:02,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:04,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:06,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:08,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:10,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:11,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:11,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:12,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:12,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:14,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:15,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:23:15,195 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:23:15,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:15,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:16,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:17,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:17,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:18,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:19,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:19,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:20,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:20,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:21,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:22,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:23,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:23,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:24,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:26,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:28,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:28,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:29,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:23:30,445 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:23:30,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:30,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:31,316 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:23:32,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:33,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:33,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:36,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:36,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:37,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:39,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:41,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:44,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:44,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:45,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:45,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:23:45,667 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:23:46,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:46,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:47,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:48,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:49,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:49,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:53,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:54,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:54,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:55,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:55,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:57,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:23:59,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:23:59,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:01,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:01,390 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:24:01,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:24:01,424 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:24:04,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:04,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:24:05,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:07,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:09,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:13,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:15,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:16,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:24:16,647 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:24:17,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:19,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:23,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:25,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:27,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:29,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:31,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:31,463 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:24:31,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:24:31,915 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:24:33,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:35,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:37,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:39,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:41,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:43,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:45,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:47,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:24:47,141 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:24:47,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:49,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:51,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:53,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:55,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:57,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:24:59,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:01,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:01,539 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:25:02,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:25:02,464 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:25:02,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:02,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:03,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:05,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:05,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:05,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:06,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:07,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:07,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:08,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:09,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:09,762 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 14:25:10,024 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 14:25:10,095 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 14:25:10,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 14:25:11,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:11,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:11,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:12,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:14,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:16,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:17,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:17,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:25:17,807 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:25:18,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:18,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:19,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:20,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:21,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:22,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:22,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:24,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:25,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:26,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:26,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:28,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:29,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:30,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:30,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:31,650 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:25:31,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:31,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:32,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:25:33,031 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:25:33,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:33,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:35,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:38,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:38,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:39,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:41,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:41,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:41,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:44,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:45,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:46,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:46,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:47,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:48,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:48,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:25:48,392 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:25:49,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:49,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:51,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:52,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:54,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:25:54,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:54,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:56,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:25:58,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:00,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:01,724 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:26:02,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:03,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:26:03,619 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:26:04,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:06,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:10,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:12,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:14,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:16,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:18,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:18,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:26:18,908 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:26:20,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:22,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:24,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:26,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:28,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:30,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:31,806 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:26:32,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:34,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:26:34,190 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:26:34,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:36,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:38,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:40,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:42,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:44,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:46,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:48,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:49,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:26:49,449 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:26:50,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:52,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:54,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:54,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:55,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:55,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:56,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:56,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:57,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:58,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:26:59,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:26:59,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:01,886 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:27:02,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:02,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:03,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:04,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:04,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:27:04,706 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:27:08,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:09,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:09,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:10,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:11,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:12,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:12,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:13,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:14,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:16,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:16,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:19,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:27:19,928 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:27:20,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:20,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:21,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:21,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:23,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:24,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:25,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:25,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:26,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:26,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:28,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:30,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:31,964 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:27:32,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:32,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:32,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:27:35,254 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:27:35,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:35,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:36,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:37,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:38,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:38,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:39,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:41,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:41,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:43,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:44,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:44,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:45,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:47,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:49,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:49,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:27:50,564 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:27:50,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:50,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:51,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:51,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:27:53,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:55,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:27:57,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:01,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:02,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:28:03,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:05,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:05,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:28:05,786 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:28:07,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:09,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:11,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:13,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:15,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:17,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:19,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:21,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:28:21,115 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:28:21,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:23,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:25,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:27,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:29,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:31,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:32,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:28:33,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:35,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:36,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:28:36,336 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:28:37,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:39,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:41,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:43,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:44,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:44,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:45,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:45,815 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 14:28:45,938 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 14:28:45,997 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 14:28:46,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 14:28:47,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:48,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:48,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:48,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:50,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:51,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:28:51,665 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:28:53,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:28:55,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:55,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:28:57,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:01,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:01,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:02,202 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:29:02,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:02,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:04,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:05,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:06,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:06,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:29:06,947 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:29:08,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:08,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:09,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:09,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:11,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:11,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:12,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:13,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:14,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:15,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:16,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:16,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:17,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:18,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:18,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:19,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:20,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:21,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:22,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:29:22,187 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:29:22,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:22,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:25,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:25,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:26,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:27,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:29,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:29,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:30,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:31,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:32,278 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:29:32,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:32,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:33,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:33,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:34,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:35,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:36,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:36,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:37,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:29:37,424 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:29:37,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:40,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:40,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:41,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:42,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:42,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:43,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:44,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:44,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:45,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:45,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:46,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:29:47,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:49,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:51,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:52,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:29:52,647 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:29:53,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:57,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:29:59,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:01,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:02,350 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:30:03,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:05,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:07,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:07,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:30:07,936 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:30:09,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:11,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:13,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:15,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:17,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:19,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:21,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:23,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:30:23,194 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:30:23,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:26,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:28,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:30,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:32,426 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:30:32,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:32,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:32,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:33,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:34,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:36,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:38,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:38,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:30:38,458 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:30:41,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:41,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:42,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:42,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:43,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:46,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:47,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:47,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:48,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:48,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:52,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:53,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:30:53,695 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:30:56,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:56,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:30:57,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:57,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:30:59,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:00,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:00,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:02,501 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:31:02,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:02,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:03,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:05,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:06,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:06,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:08,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:09,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:31:09,015 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:31:09,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:09,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:12,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:12,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:14,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:15,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:15,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:18,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:20,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:22,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:23,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:23,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:31:24,243 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:31:24,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:24,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:25,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:26,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:27,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:27,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:28,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:29,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:29,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:30,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:30,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:32,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:32,578 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:31:34,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:34,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:34,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:36,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:37,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:37,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:39,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:31:39,466 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:31:40,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:41,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:41,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:42,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:43,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:43,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:31:44,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:46,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:48,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:50,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:52,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:54,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:54,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:31:54,688 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:31:57,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:31:59,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:01,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:02,653 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:32:03,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:05,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:07,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:09,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:09,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:32:09,937 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:32:11,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:13,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:15,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:17,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:19,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:20,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 14:32:20,725 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 14:32:20,961 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 14:32:21,040 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 14:32:21,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:21,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 14:32:23,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:25,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:32:25,272 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:32:25,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:27,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:29,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:31,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:33,085 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:32:33,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:35,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:37,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:39,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:40,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:32:40,627 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:32:41,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:43,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:45,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:47,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:49,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:51,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:53,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:55,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:55,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:32:55,924 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:32:57,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:32:59,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:01,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:03,163 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:33:03,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:05,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:08,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:10,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:11,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:33:11,734 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:33:12,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:14,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:16,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:18,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:20,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:22,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:24,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:26,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:26,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:33:26,962 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:33:28,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:30,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:32,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:33,237 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:33:34,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:36,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:38,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:40,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:42,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:33:42,225 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:33:42,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:44,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:46,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:48,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:50,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:52,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:54,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:56,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:33:57,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:33:57,549 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:33:58,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:00,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:02,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:03,316 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:34:04,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:06,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:08,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:10,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:12,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:34:12,774 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:34:13,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:15,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:17,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:19,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:21,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:23,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:25,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:27,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:28,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:34:28,012 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:34:29,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:31,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:33,392 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:34:33,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:35,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:37,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:39,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:41,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:43,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:34:43,353 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:34:43,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:45,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:47,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:49,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:51,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:53,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:55,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:57,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:34:58,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:34:58,764 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:34:59,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:01,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:03,462 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:35:03,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:05,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:07,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:09,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:11,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:13,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:13,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:35:13,983 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:35:15,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:17,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:19,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:21,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:23,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:25,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:27,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:29,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:35:29,417 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:35:29,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:31,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:33,535 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:35:33,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:35,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:37,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:39,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:41,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:43,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:44,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:35:44,651 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:35:46,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:48,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:50,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:52,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:54,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:56,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:58,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:35:59,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:35:59,967 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:36:00,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:02,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:03,613 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:36:04,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:06,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:08,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:10,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:12,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:14,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:15,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:36:15,190 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:36:16,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:18,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:20,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:22,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:24,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:26,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:28,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:30,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:36:30,514 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:36:30,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:32,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:33,690 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:36:34,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:36,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:38,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:40,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:42,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:44,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:45,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:36:45,802 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:36:46,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:48,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:50,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:52,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:54,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:57,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:36:59,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:01,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:37:01,126 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:37:01,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:03,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:03,765 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:37:05,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:07,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:09,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:11,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:13,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:15,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:16,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:37:16,371 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:37:17,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:19,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:21,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:23,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:25,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:27,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:29,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:31,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:37:31,594 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:37:31,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:33,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:33,837 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:37:35,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:37,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:39,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:41,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:43,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:45,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:46,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:37:46,911 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:37:47,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:49,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:51,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:53,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:55,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:57,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:37:59,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:01,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:02,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:38:02,176 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:38:03,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:03,912 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:38:05,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:07,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:09,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:11,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:13,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:15,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:17,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:38:17,395 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:38:17,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:19,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:22,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:24,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:26,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:28,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:30,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:32,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:32,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:38:32,743 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:38:33,987 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:38:34,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:36,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:38,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:40,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:42,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:44,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:46,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:48,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:38:48,009 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:38:48,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:50,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:52,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:54,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:56,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:38:58,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:00,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:02,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:03,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:39:03,345 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:39:04,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:39:04,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:06,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:08,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:10,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:12,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:14,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:16,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:18,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:39:18,572 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:39:18,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:20,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:22,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:24,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:26,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:28,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:30,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:32,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:33,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:39:33,796 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:39:34,138 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:39:34,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:36,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:38,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:40,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:42,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:44,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:47,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:49,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:39:49,055 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:39:49,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:51,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:53,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:55,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:57,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:39:59,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:01,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:03,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:04,217 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:40:04,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:40:04,525 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:40:05,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:07,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:09,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:11,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:13,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:15,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:17,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:19,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:19,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:40:19,748 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:40:21,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:23,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:25,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:27,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:29,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:31,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:33,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:34,294 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:40:34,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:40:34,976 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:40:35,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:37,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:39,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:41,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:43,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:45,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:47,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:49,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:50,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:40:50,289 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:40:51,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:53,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:55,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:57,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:40:59,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:01,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:03,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:04,373 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:41:05,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:41:05,532 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:41:06,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:08,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:10,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:12,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:14,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:16,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:18,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:20,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:20,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:41:20,797 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:41:22,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:24,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:26,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:28,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:30,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:32,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:34,448 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:41:34,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:36,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:41:36,123 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:41:36,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:38,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:40,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:42,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:44,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:46,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:48,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:50,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:51,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:41:51,430 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:41:52,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:54,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:56,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:41:58,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:00,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:02,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:04,525 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:42:04,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:06,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:42:06,700 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:42:06,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:08,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:10,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:12,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:14,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:16,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:18,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:20,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:21,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:42:21,929 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:42:23,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:25,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:27,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:29,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:31,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:33,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:34,601 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:42:35,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:37,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:42:37,297 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:42:37,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:39,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:41,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:43,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:45,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:47,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:49,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:51,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:52,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:42:52,623 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:42:53,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:55,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:57,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:42:59,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:01,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:03,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:04,679 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:43:05,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:07,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:07,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:43:07,905 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:43:09,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:11,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:13,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:15,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:17,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:19,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:21,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:23,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:43:23,134 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:43:23,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:25,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:27,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:30,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:32,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:34,753 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:43:34,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:36,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:38,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:43:38,425 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:43:38,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:40,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:42,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:44,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:46,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:48,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:50,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:52,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:53,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:43:53,719 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:43:54,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:56,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:43:58,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:00,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:02,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:04,825 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:44:04,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:06,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:08,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:09,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:44:09,018 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:44:10,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:12,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:14,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:16,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:18,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:20,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:22,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:24,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:44:24,412 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:44:24,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:26,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:28,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:30,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:32,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:34,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:34,902 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:44:36,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:38,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:39,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:44:39,637 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:44:40,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:42,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:44,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:46,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:48,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:50,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:53,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:54,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:44:54,964 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:44:55,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:57,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:44:59,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:01,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:03,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:04,970 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:45:05,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:07,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:09,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:10,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:45:10,176 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:45:11,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:13,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:15,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:17,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:19,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:21,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:23,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:25,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:45:25,418 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:45:25,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:27,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:29,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:31,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:33,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:35,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:45:35,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:37,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:39,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:40,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:45:40,640 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:45:41,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:43,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:45,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:48,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:50,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:52,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:54,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:55,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:45:55,928 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:45:56,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:45:58,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:00,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:02,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:04,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:05,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:46:06,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:08,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:10,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:11,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:46:11,276 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:46:12,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:14,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:16,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:18,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:20,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:22,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:24,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:26,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:46:26,692 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:46:26,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:28,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:30,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:32,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:34,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:35,172 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:46:36,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:38,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:40,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:42,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:46:42,009 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:46:42,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:44,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:46,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:48,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:50,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:52,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:54,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:56,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:46:57,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:46:57,246 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:46:58,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:00,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:02,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:04,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:05,244 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:47:06,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:08,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:10,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:12,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:47:12,578 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:47:13,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:15,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:17,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:19,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:21,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:23,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:25,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:27,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:47:27,858 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:47:27,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:29,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:31,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:33,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:35,308 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:47:35,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:37,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:39,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:41,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:43,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:47:43,187 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:47:43,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:45,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:47,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:49,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:51,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:53,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:55,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:57,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:47:58,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:47:58,582 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:47:59,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:01,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:03,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:05,380 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:48:05,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:07,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:09,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:11,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:13,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:48:13,803 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:48:13,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:15,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:17,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:19,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:21,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:23,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:25,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:27,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:29,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:48:29,103 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:48:29,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:31,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:33,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:35,451 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:48:35,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:37,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:39,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:41,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:43,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:44,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:48:44,326 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:48:45,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:47,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:49,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:51,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:53,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:55,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:58,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:48:59,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:48:59,631 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:49:00,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:02,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:04,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:05,520 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:49:06,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:08,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:10,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:12,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:14,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:49:14,890 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:49:14,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:16,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:18,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:20,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:22,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:24,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:26,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:28,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:30,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:49:30,236 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:49:30,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:32,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:34,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:35,595 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:49:36,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:38,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:40,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:42,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:44,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:45,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:49:45,458 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:49:46,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:48,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:50,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:52,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:54,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:57,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:49:59,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:00,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:50:00,741 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:50:01,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:03,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:05,676 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:50:05,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:07,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:09,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:11,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:13,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:15,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:16,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:50:16,046 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:50:17,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:19,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:21,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:23,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:25,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:27,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:29,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:31,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:50:31,341 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:50:31,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:33,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:35,744 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:50:35,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:37,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:39,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:41,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:43,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:46,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:46,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:50:46,751 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:50:48,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:50,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:52,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:54,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:56,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:50:58,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:00,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:01,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:51:01,993 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:51:02,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:04,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:05,812 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:51:06,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:08,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:10,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:12,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:14,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:16,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:17,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:51:17,278 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:51:18,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:20,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:23,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:25,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:27,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:29,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:31,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:32,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:51:32,538 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:51:33,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:35,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:35,886 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:51:37,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:39,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:41,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:43,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:45,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:47,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:47,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:51:47,811 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:51:49,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:51,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:53,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:55,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:57,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:51:59,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:01,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:03,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:03,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:52:03,127 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:52:05,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:05,956 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:52:07,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:09,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:11,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:13,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:15,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:17,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:18,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:52:18,352 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:52:19,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:21,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:23,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:25,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:27,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:29,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:32,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:33,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:52:33,656 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:52:34,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:36,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:52:36,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:38,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:40,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:42,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:44,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:46,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:48,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:49,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:52:49,023 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:52:50,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:52,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:54,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:56,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:52:58,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:00,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:02,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:04,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:04,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:53:04,278 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:53:06,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:06,090 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:53:08,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:10,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:12,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:14,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:16,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:18,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:19,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:53:19,567 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:53:20,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:22,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:24,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:26,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:28,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:30,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:32,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:34,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:34,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:53:34,857 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:53:36,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:36,152 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:53:38,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:40,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:42,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:44,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:46,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:48,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:50,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:50,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:53:50,091 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:53:52,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:54,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:56,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:53:58,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:01,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:03,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:05,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:05,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:54:05,355 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:54:06,214 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:54:07,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:09,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:11,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:13,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:15,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:17,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:19,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:20,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:54:20,729 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:54:21,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:23,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:25,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:27,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:29,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:31,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:33,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:35,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:36,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:54:36,058 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:54:36,283 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:54:37,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:39,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:41,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:43,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:45,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:47,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:49,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:51,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:51,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:54:51,322 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:54:53,100 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:55,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:57,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:54:59,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:01,103 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:03,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:05,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:06,352 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:55:06,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:55:06,547 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:55:07,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:09,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:11,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:13,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:15,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:17,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:19,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:21,112 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:21,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:55:21,870 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:55:23,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:25,114 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:27,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:29,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:31,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:33,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:35,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:36,427 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:55:37,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:55:37,129 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:55:38,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:40,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:42,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:44,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:46,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:48,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:50,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:52,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:52,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:55:52,409 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:55:54,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:56,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:55:58,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:00,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:02,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:04,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:06,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:06,501 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:56:07,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:56:07,634 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:56:08,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:10,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:12,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:14,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:16,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:18,140 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:20,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:22,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:22,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:56:22,857 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:56:24,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:26,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:28,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:30,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:32,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:34,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:36,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:36,574 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:56:38,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:56:38,079 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:56:38,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:40,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:42,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:44,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:46,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:48,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:50,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:52,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:53,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:56:53,301 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:56:54,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:56,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:56:58,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:00,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:02,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:04,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:06,648 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:57:07,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:08,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:57:08,608 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:57:09,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:11,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:13,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:15,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:17,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:19,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:21,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:23,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:23,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:57:23,841 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:57:25,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:27,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:29,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:31,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:33,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:35,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:36,713 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:57:37,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:39,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:57:39,122 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:57:39,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:41,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:43,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:45,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:47,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:49,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:51,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:53,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:54,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:57:54,345 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:57:55,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:57,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:57:59,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:01,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:03,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:05,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:06,784 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:58:07,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:09,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:09,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:58:09,659 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:58:11,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:13,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:15,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:17,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:19,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:21,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:23,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:25,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:58:25,005 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:58:25,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:27,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:29,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:31,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:33,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:35,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:36,857 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:58:37,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:39,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:40,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:58:40,244 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:58:41,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:44,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:46,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:48,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:50,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:52,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:54,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:55,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:58:55,467 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:58:56,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:58:58,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:00,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:02,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:04,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:06,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:06,919 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:59:08,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:10,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:10,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:59:10,690 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:59:12,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:14,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:16,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:18,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:20,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:22,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:24,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:25,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:59:25,912 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:59:26,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:28,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:30,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:32,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:34,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:36,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:36,995 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 14:59:38,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:40,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:41,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:59:41,217 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:59:42,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:44,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:46,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:48,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:50,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:52,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:54,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:56,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 14:59:56,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 14:59:56,440 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 14:59:58,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:00,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:02,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:04,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:06,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:07,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:00:08,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:11,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:11,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:00:11,715 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:00:13,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:15,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:17,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:19,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:21,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:23,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:25,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:27,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:00:27,103 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:00:27,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:29,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:31,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:33,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:35,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:37,134 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:00:37,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:39,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:41,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:42,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:00:42,348 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:00:43,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:45,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:47,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:49,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:51,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:53,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:55,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:57,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:00:57,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:00:57,647 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:00:59,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:01,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:03,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:05,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:07,199 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:01:07,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:09,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:11,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:12,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:01:12,900 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:01:13,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:15,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:17,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:19,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:21,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:23,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:25,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:27,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:28,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:01:28,160 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:01:29,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:31,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:33,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:35,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:37,272 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:01:37,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:39,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:42,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:43,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:01:43,409 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:01:44,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:46,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:48,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:50,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:52,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:54,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:56,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:58,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:01:58,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:01:58,708 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:02:00,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:02,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:04,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:06,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:07,332 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:02:08,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:10,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:12,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:13,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:02:13,968 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:02:14,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:16,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:18,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:20,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:22,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:24,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:26,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:28,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:29,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:02:29,257 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:02:30,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:32,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:34,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:36,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:37,391 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:02:38,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:40,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:42,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:44,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:02:44,485 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:02:46,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:48,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:50,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:53,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:55,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:57,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:59,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:02:59,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:02:59,894 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:03:01,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:03,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:05,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:07,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:07,464 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:03:09,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:11,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:13,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:15,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:03:15,201 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:03:15,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:17,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:19,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:21,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:23,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:25,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:27,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:29,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:30,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:03:30,425 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:03:31,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:33,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:35,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:37,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:37,539 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:03:39,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:41,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:43,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:45,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:45,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:03:45,700 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:03:47,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:49,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:51,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:53,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:55,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:57,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:03:59,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:00,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:04:00,924 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:04:02,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:04,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:06,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:07,616 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:04:08,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:10,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:12,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:14,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:16,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:04:16,228 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:04:16,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:18,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:20,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:22,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:24,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:26,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:28,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:30,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:31,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:04:31,452 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:04:32,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:34,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:36,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:37,688 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:04:38,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:40,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:42,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:44,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:46,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:46,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:04:46,672 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:04:48,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:51,355 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:53,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:55,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:57,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:04:59,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:01,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:02,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:05:02,093 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:05:03,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:05,361 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:07,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:07,760 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:05:09,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:11,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:13,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:15,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:17,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:17,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:05:17,398 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:05:19,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:21,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:23,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:25,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:27,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:29,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:31,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:32,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:05:32,691 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:05:33,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:35,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:37,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:37,835 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:05:39,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:41,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:43,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:45,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:47,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:47,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:05:47,991 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:05:49,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:51,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:53,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:55,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:57,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:05:59,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:01,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:03,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:06:03,211 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:06:03,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:05,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:07,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:07,910 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:06:09,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:11,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:14,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:16,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:18,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:18,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:06:18,471 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:06:20,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:22,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:24,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:26,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:28,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:30,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:32,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:33,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:06:33,692 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:06:34,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:36,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:37,986 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:06:38,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:40,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:42,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:44,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:46,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:48,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:49,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:06:49,065 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:06:50,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:52,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:54,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:56,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:06:58,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:00,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:02,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:04,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:07:04,313 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:07:04,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:06,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:08,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:07:08,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:10,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:12,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:14,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:16,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:18,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:19,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:07:19,537 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:07:20,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:22,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:24,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:26,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:28,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:30,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:33,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:34,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:07:34,881 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:07:35,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:37,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:38,140 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:07:39,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:41,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:43,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:45,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:47,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:49,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:50,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:07:50,114 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:07:51,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:53,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:55,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:57,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:07:59,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:01,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:03,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:05,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:08:05,410 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:08:05,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:07,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:08,214 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:08:09,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:11,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:13,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:15,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:17,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:19,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:20,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:08:20,732 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:08:21,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:23,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:25,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:27,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:29,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:31,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:33,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:35,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:35,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:08:35,947 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:08:37,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:38,286 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:08:39,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:41,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:43,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:45,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:47,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:49,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:51,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:08:51,237 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:08:51,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:53,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:55,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:57,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:08:59,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:01,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:03,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:05,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:06,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:09:06,575 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:09:07,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:08,361 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:09:09,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:11,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:13,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:15,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:17,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:19,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:21,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:21,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:09:21,903 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:09:23,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:25,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:28,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:30,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:32,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:34,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:36,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:37,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:09:37,137 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:09:38,437 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:09:38,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:40,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:42,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:44,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:46,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:48,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:50,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:52,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:09:52,362 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:09:52,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:54,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:56,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:09:58,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:00,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:02,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:04,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:06,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:07,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:10:07,581 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:10:08,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:08,513 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:10:10,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:12,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:14,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:16,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:18,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:20,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:22,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:22,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:10:22,932 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:10:24,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:26,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:28,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:30,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:32,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:34,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:36,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:38,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:10:38,224 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:10:38,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:38,585 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:10:41,506 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:43,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:45,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:47,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:49,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:51,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:53,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:10:53,456 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:10:53,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:55,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:57,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:10:59,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:01,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:03,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:05,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:07,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:08,659 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:11:08,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:11:08,681 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:11:09,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:11,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:13,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:15,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:17,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:19,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:21,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:23,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:23,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:11:23,951 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:11:25,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:27,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:29,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:31,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:33,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:35,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:37,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:38,734 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:11:39,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:11:39,226 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:11:39,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:41,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:43,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:45,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:47,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:49,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:51,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:53,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:54,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:11:54,539 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:11:55,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:57,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:11:59,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:02,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:04,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:06,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:08,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:08,813 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:12:09,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:12:09,875 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:12:10,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:12,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:14,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:16,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:18,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:20,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:22,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:24,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:25,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:12:25,100 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:12:26,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:28,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:30,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:32,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:34,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:36,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:38,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:38,896 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:12:40,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:12:40,471 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:12:40,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:42,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:44,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:46,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:48,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:50,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:52,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:54,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:55,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:12:55,808 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:12:56,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:12:58,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:00,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:02,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:04,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:06,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:08,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:08,972 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:13:10,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:11,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:13:11,030 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:13:12,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:14,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:16,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:18,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:20,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:22,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:24,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:26,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:13:26,290 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:13:26,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:28,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:30,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:32,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:34,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:36,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:38,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:39,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:13:40,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:41,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:13:41,550 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:13:42,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:44,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:46,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:48,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:50,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:52,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:54,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:56,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:13:56,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:13:56,775 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:13:58,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:00,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:02,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:04,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:06,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:08,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:09,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:14:10,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:11,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:14:11,999 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:14:12,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:15,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:17,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:19,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:21,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:23,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:25,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:27,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:14:27,221 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:14:27,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:29,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:31,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:33,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:35,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:37,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:39,193 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:14:39,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:41,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:42,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:14:42,610 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:14:43,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:45,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:47,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:49,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:51,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:53,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:55,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:57,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:14:57,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:14:57,904 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:14:59,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:01,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:03,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:05,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:07,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:09,267 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:15:09,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:11,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:13,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:15:13,131 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:15:13,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:15,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:17,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:19,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:21,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:23,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:25,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:27,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:28,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:15:28,370 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:15:29,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:31,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:34,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:36,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:38,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:39,340 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:15:40,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:42,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:43,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:15:43,751 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:15:44,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:46,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:48,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:50,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:52,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:54,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:56,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:58,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:15:59,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:15:59,086 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:16:00,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:02,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:04,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:06,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:08,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:09,414 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:16:10,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:12,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:14,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:16:14,335 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:16:14,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:16,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:18,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:20,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:22,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:24,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:26,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:28,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:29,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:16:29,653 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:16:30,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:32,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:34,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:36,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:38,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:39,490 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:16:40,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:42,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:44,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:44,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:16:44,878 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:16:46,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:48,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:50,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:52,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:54,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:56,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:16:58,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:00,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:17:00,105 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:17:00,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:02,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:04,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:06,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:08,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:09,568 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:17:10,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:12,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:15,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:17:15,325 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:17:15,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:17,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:19,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:21,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:23,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:25,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:27,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:29,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:30,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:17:30,541 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:17:31,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:33,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:35,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:37,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:39,642 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:17:39,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:41,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:43,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:45,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:45,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:17:45,858 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:17:47,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:49,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:51,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:53,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:55,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:57,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:17:59,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:01,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:18:01,102 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:18:01,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:03,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:05,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:07,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:09,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:09,715 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:18:11,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:13,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:15,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:16,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:18:16,326 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:18:17,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:19,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:21,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:23,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:25,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:27,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:29,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:31,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:18:31,537 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:18:31,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:33,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:35,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:37,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:39,789 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:18:40,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:42,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:44,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:46,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:46,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:18:46,828 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:18:48,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:50,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:52,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:54,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:56,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:18:58,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:00,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:02,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:19:02,049 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:19:02,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:04,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:06,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:08,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:09,863 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:19:10,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:12,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:14,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:16,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:17,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:19:17,335 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:19:18,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:20,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:22,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:24,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:26,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:28,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:30,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:32,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:19:32,597 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:19:32,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:34,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:36,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:38,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:39,938 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:19:40,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:42,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:44,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:46,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:47,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:19:47,869 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:19:48,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:50,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:52,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:54,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:56,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:19:58,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:00,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:02,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:03,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:20:03,089 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:20:04,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:06,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:08,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:10,013 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:20:10,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:12,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:14,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:16,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:18,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:20:18,313 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:20:18,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:20,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:22,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:24,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:26,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:28,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:30,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:32,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:33,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:20:33,573 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:20:34,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:36,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:38,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:40,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:20:41,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:43,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:45,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:47,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:48,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:20:48,921 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:20:49,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:51,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:53,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:55,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:57,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:20:59,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:01,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:03,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:04,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:21:04,200 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:21:05,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:07,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:09,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:10,165 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:21:11,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:13,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:15,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:17,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:19,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:21:19,507 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:21:19,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:21,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:23,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:25,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:27,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:29,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:31,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:33,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:34,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:21:34,729 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:21:35,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:37,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:39,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:40,243 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:21:41,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:43,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:45,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:47,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:49,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:50,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:21:50,056 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:21:51,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:53,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:55,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:57,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:21:59,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:01,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:03,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:05,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:22:05,324 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:22:05,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:07,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:09,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:10,319 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:22:12,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:14,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:16,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:18,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:20,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:22:20,545 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:22:20,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:22,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:24,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:26,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:28,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:30,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:32,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:34,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:35,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:22:35,792 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:22:36,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:38,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:40,395 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:22:40,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:42,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:44,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:46,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:48,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:50,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:51,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:22:51,117 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:22:52,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:54,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:56,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:22:58,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:00,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:02,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:04,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:06,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:23:06,364 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:23:06,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:08,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:10,471 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:23:10,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:12,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:14,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:16,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:18,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:20,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:21,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:23:21,587 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:23:22,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:24,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:26,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:28,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:30,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:32,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:34,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:36,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:23:36,812 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:23:36,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:39,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:40,556 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:23:41,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:43,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:45,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:47,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:49,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:51,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:52,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:23:52,194 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:23:53,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:55,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:57,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:23:59,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:01,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:03,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:05,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:07,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:24:07,526 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:24:07,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:09,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:10,632 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:24:11,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:13,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:15,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:17,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:19,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:21,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:22,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:24:22,751 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:24:23,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:25,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:27,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:29,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:31,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:33,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:35,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:37,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:39,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:40,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:24:40,330 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:24:40,710 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:24:41,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:43,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:45,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:47,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:49,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:51,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:53,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:55,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:24:55,554 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:24:55,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:57,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:24:59,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:01,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:03,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:05,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:07,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:09,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:10,787 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:25:10,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:25:10,802 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:25:11,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:13,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:16,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:18,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:20,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:22,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:24,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:26,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:25:26,083 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:25:26,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:28,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:30,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:32,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:34,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:36,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:38,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:40,852 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:25:40,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:41,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:25:41,365 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:25:42,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:44,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:46,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:48,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:50,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:52,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:54,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:56,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:25:56,652 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:25:56,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:25:58,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:00,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:02,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:04,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:06,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:08,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:10,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:10,926 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:26:11,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:26:11,944 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:26:12,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:14,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:16,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:18,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:20,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:22,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:24,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:26,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:27,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:26:27,170 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:26:28,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:30,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:32,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:34,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:36,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:38,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:40,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:41,001 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:26:42,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:26:42,499 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:26:42,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:44,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:46,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:48,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:50,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:52,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:54,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:56,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:26:57,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:26:57,792 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:26:58,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:00,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:02,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:04,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:06,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:08,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:10,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:11,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:27:12,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:13,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:27:13,091 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:27:14,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:16,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:18,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:20,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:22,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:24,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:26,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:28,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:27:28,343 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:27:29,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:31,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:33,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:35,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:37,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:39,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:41,149 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:27:41,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:43,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:27:43,579 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:27:43,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:45,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:47,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:49,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:51,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:53,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:55,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:57,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:27:58,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:27:58,947 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:27:59,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:01,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:03,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:05,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:07,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:09,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:11,225 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:28:11,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:13,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:14,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:28:14,171 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:28:15,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:17,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:19,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:21,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:23,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:25,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:27,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:29,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:28:29,460 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:28:29,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:31,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:33,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:35,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:37,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:39,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:41,300 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:28:41,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:43,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:44,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:28:44,738 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:28:45,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:47,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:49,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:51,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:54,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:56,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:58,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:28:59,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:28:59,953 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:29:00,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:02,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:04,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:06,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:08,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:10,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:11,373 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:29:12,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:15,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:15,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:29:15,226 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:29:17,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:19,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:21,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:23,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:25,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:27,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:29,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:30,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:29:30,465 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:29:31,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:33,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:35,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:37,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:39,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:41,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:41,446 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:29:43,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:45,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:45,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:29:45,727 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:29:47,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:49,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:51,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:53,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:55,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:57,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:29:59,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:00,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:30:00,959 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:30:01,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:03,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:05,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:07,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:09,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:11,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:11,517 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:30:13,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:15,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:16,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:30:16,242 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:30:17,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:19,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:21,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:23,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:25,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:27,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:29,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:31,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:30:31,546 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:30:32,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:34,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:36,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:38,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:40,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:41,594 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:30:42,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:44,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:46,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:46,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:30:46,772 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:30:48,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:50,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:52,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:54,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:56,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:30:58,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:00,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:02,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:31:02,041 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:31:02,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:04,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:06,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:08,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:10,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:11,671 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:31:12,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:14,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:16,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:17,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:31:17,365 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:31:18,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:20,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:22,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:24,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:26,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:28,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:30,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:32,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:32,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:31:32,731 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:31:34,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:36,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:38,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:40,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:41,736 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:31:42,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:44,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:46,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:47,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:31:47,988 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:31:48,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:50,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:52,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:55,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:57,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:31:59,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:01,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:03,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:03,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:32:03,310 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:32:05,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:07,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:09,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:11,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:11,812 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:32:13,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:15,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:17,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:18,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:32:18,545 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:32:19,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:21,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:23,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:25,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:27,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:29,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:31,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:33,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:33,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:32:33,853 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:32:35,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:37,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:39,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:41,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:41,889 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:32:43,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:45,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:47,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:49,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:32:49,088 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:32:49,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:51,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:53,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:55,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:57,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:32:59,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:01,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:03,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:04,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:33:04,312 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:33:05,100 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:07,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:09,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:11,103 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:11,966 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:33:13,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:15,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:17,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:19,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:19,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:33:19,606 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:33:22,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:24,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:26,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:28,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:30,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:32,112 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:34,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:34,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:33:34,831 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:33:36,114 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:38,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:40,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:42,041 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:33:42,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:44,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:46,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:48,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:50,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:50,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:33:50,156 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:33:52,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:54,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:56,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:33:58,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:00,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:02,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:04,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:05,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:34:05,378 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:34:06,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:08,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:10,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:12,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:34:12,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:14,132 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:16,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:18,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:20,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:20,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:34:20,671 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:34:22,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:24,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:26,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:28,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:30,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:32,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:34,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:35,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:34:35,965 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:34:37,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:39,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:41,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:42,194 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:34:43,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:45,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:47,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:49,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:51,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:51,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:34:51,280 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:34:53,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:55,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:57,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:34:59,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:01,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:03,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:05,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:06,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:35:06,505 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:35:07,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:09,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:11,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:12,265 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:35:13,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:15,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:17,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:19,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:21,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:21,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:35:21,829 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:35:23,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:25,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:27,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:29,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:31,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:33,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:35,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:37,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:37,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:35:37,173 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:35:39,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:41,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:42,336 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:35:43,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:45,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:47,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:49,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:51,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:52,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:35:52,397 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:35:53,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:55,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:35:58,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:00,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:02,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:04,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:06,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:07,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:36:07,664 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:36:08,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:10,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:12,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:12,412 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:36:14,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:16,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:18,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:20,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:22,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:22,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:36:22,885 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:36:24,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:26,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:28,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:30,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:32,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:34,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:36,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:38,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:36:38,105 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:36:38,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:40,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:42,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:42,488 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:36:44,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:46,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:48,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:50,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:52,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:53,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:36:53,327 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:36:54,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:56,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:36:59,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:01,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:03,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:05,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:07,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:08,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:37:08,640 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:37:09,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:11,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:12,560 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:37:13,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:15,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:17,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:19,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:21,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:23,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:23,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:37:23,963 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:37:25,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:27,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:29,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:31,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:33,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:35,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:37,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:39,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:39,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:37:39,280 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:37:41,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:42,632 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:37:43,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:45,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:47,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:49,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:51,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:53,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:54,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:37:54,591 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:37:55,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:37:58,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:00,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:02,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:04,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:06,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:08,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:09,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:38:09,894 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:38:10,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:12,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:12,706 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:38:14,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:16,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:18,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:20,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:22,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:24,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:25,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:38:25,145 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:38:26,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:28,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:30,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:32,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:34,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:36,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:38,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:40,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:40,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:38:40,400 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:38:42,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:42,785 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:38:44,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:46,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:48,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:50,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:52,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:54,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:55,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:38:55,711 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:38:56,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:38:58,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:00,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:02,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:05,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:07,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:09,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:10,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:39:10,992 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:39:11,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:12,861 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:39:13,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:15,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:17,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:19,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:21,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:23,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:25,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:26,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:39:26,240 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:39:27,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:29,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:31,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:33,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:35,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:37,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:39,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:41,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:41,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:39:41,463 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:39:42,935 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:39:43,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:45,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:47,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:49,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:51,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:53,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:55,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:56,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:39:56,787 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:39:57,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:39:59,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:01,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:03,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:05,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:08,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:10,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:12,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:40:12,013 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:40:12,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:13,010 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:40:14,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:16,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:18,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:20,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:22,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:24,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:26,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:27,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:40:27,241 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:40:28,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:30,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:32,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:34,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:36,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:38,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:40,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:42,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:42,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:40:42,559 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:40:43,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:40:44,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:46,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:48,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:50,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:52,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:54,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:56,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:40:57,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:40:57,789 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:40:58,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:00,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:02,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:04,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:06,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:08,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:10,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:12,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:13,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:41:13,013 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:41:13,515 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:41:14,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:16,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:18,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:20,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:22,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:24,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:26,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:28,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:28,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:41:28,572 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:41:30,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:32,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:34,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:36,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:38,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:40,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:42,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:43,245 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:41:43,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:41:43,931 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:41:44,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:46,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:48,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:50,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:52,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:54,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:56,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:58,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:41:59,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:41:59,243 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:42:00,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:02,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:04,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:06,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:08,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:10,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:12,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:13,320 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:42:14,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:14,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:42:14,467 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:42:16,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:18,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:20,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:22,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:24,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:26,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:28,355 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:29,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:42:29,739 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:42:30,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:32,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:34,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:37,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:39,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:41,361 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:43,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:43,397 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:42:45,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:42:45,025 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:42:45,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:47,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:49,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:51,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:53,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:42:55,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:43:00,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:43:00,243 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:43:13,473 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:43:15,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:43:15,515 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:43:30,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:43:30,763 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:43:43,549 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:43:46,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:43:46,043 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:44:01,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:44:01,333 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:44:09,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:44:09,494 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 15:44:09,549 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 15:44:09,633 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 15:44:10,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 15:44:11,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:44:13,647 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:44:16,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:44:16,598 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:44:17,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:44:31,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:44:31,823 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:44:33,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:44:43,727 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:44:47,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:44:47,050 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:45:02,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:45:02,274 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:45:13,805 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:45:17,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:45:17,524 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:45:32,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:45:32,750 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:45:43,884 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:45:48,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:45:48,088 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:46:03,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:46:03,314 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:46:13,961 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:46:18,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:46:18,537 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:46:33,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:46:33,765 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:46:44,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:46:48,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:46:48,989 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:47:04,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:47:04,214 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:47:14,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:47:19,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:47:19,441 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:47:34,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:47:34,801 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:47:44,194 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:47:50,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:47:50,033 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:48:05,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:48:05,258 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:48:14,275 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:48:20,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:48:20,485 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:48:25,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:48:33,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:48:34,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:34,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:35,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:48:35,737 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:48:37,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:48:38,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:38,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:40,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:48:41,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:41,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:42,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:44,351 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:48:44,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:48:45,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:45,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:48,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:48:50,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:50,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:48:51,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:48:51,033 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:48:53,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:53,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:48:54,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:54,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:48:56,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:56,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:58,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:48:59,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:00,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:00,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:02,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:03,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:03,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:05,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:49:06,260 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:49:06,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:06,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:07,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:07,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:08,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:08,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:09,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:10,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:11,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:12,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:13,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:13,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,428 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:49:14,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:14,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:16,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:18,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:19,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:19,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:20,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:20,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:49:21,484 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:49:21,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:21,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:22,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:24,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:25,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:25,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:26,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:27,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:27,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:28,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:30,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:30,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:30,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:32,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:34,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:35,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:35,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:36,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:36,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:49:36,708 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:49:38,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:40,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:40,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:40,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:49:42,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:44,498 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:49:44,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:46,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:48,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:50,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:51,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:49:51,946 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:49:52,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:54,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:56,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:49:58,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:00,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:02,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:04,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:06,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:07,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:50:07,736 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:50:08,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:10,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:10,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:10,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:12,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:14,563 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:50:18,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:22,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:22,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:23,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:50:23,102 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:50:26,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:26,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:30,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:32,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:34,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:34,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:35,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:36,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:37,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:38,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:38,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:50:38,331 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:50:40,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:40,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:41,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:43,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:44,637 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:50:46,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:46,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:47,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:47,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:49,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:50,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:50,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:52,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:50:53,555 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:50:53,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:53,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:53,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:55,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:50:57,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:57,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:58,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:50:59,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:00,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:00,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:01,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:03,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:03,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:04,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:04,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:05,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:07,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:07,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:08,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:51:08,779 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:51:09,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:09,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:10,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:10,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:11,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:13,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:14,707 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:51:14,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:14,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:15,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:16,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:16,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:17,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:18,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:18,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:19,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:20,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:20,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:23,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:24,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:51:24,096 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:51:25,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:25,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:25,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:27,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:29,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:31,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:32,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:32,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:33,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:35,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:37,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:39,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:51:39,351 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:51:39,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:41,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:43,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:44,781 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:51:45,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:47,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:49,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:51,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:53,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:54,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:51:54,653 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:51:55,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:57,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:59,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:51:59,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:51:59,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:00,007 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 15:52:00,209 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 15:52:00,318 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 15:52:00,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:00,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 15:52:01,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:07,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:09,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:52:09,990 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:52:11,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:14,950 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:52:15,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:15,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:17,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:17,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:21,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:25,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:52:25,354 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:52:25,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:25,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:25,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:28,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:29,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:29,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:31,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:31,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:31,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:33,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:35,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:37,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:38,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:38,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:40,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:52:40,577 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:52:41,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:42,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:42,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:43,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:44,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:44,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:45,020 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:52:45,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:46,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:46,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:47,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:49,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:50,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:50,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:52,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:52,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:53,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:54,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:55,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:52:55,881 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:52:58,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:52:58,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:58,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:52:59,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:00,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:01,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:01,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:02,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:04,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:04,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:06,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:06,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:06,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:07,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:08,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:09,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:09,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:10,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:11,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:53:11,106 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:53:11,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:11,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:12,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:13,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:13,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,093 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:53:15,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:15,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:16,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:18,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:18,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:20,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:22,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:23,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:23,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:24,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:26,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:53:26,327 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:53:26,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:28,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:30,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:32,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:34,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:36,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:38,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:40,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:41,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:53:41,682 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:53:42,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:44,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:45,162 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:53:46,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:48,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:53:48,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:50,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:56,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:53:56,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:53:56,906 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:54:00,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:04,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:06,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:10,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:12,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:54:12,129 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:54:12,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:12,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:14,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:15,238 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:54:15,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:15,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:16,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:17,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:17,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:20,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:23,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:23,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:24,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:24,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:26,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:27,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:54:27,351 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:54:28,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:29,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:29,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:30,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:32,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:33,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:33,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:34,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:35,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:35,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:36,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:37,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:37,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:38,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:39,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:40,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:41,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:41,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:42,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:42,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:54:42,684 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:54:43,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:43,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:44,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:44,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,311 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:54:45,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:45,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:46,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:47,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:47,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:49,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:51,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:52,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:52,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:53,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:53,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:54,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:55,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:57,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:57,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:54:57,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:54:57,932 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:54:58,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:58,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:54:59,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:00,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:00,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:01,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:02,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:02,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:03,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:04,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:04,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:05,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:06,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:06,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:07,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:08,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:08,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:09,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:11,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:13,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:55:13,149 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:55:13,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:13,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:13,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,380 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:55:15,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:15,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:15,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:17,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:19,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:21,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:23,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:25,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:27,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:28,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:55:28,370 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:55:29,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:31,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:33,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:35,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:36,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:36,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:36,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:36,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:36,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:36,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:36,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:36,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:36,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:36,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:36,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:36,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:37,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:55:37,125 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 15:55:37,347 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 15:55:37,416 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 15:55:37,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:37,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 15:55:39,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:43,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:55:43,599 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:55:43,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:45,457 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:55:47,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:51,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:55,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:57,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:55:58,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:55:58,964 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:56:01,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:05,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:06,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:06,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:07,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:09,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:09,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:11,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:13,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:14,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:56:14,254 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:56:15,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:15,518 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:56:16,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:16,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:17,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:19,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:19,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:20,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:20,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:21,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:22,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:22,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:24,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:25,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:27,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:27,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:28,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:28,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:56:29,548 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:56:29,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:29,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:30,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:31,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:31,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:33,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:34,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:34,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:35,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:36,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:37,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:37,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:38,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:39,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:39,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:40,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:40,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:42,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:42,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:43,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:43,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:43,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:43,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:43,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:43,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:43,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:43,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:44,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:44,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:56:44,765 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:56:45,581 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:56:46,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:48,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:49,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:49,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:50,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:51,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:51,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:52,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:53,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:53,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:54,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:56,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:56,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:56:57,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:57,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:56:58,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:00,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:57:00,056 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:57:00,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:00,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:02,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:02,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:04,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:06,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:06,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:06,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:08,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:10,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:12,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:14,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:15,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:57:15,327 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:57:15,656 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:57:16,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:18,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:20,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:22,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:23,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:23,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:23,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:23,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:23,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:23,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:23,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:23,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:23,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:23,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:23,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:23,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:24,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:26,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:30,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:57:30,554 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:57:30,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:36,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:38,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:42,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:45,729 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:57:45,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:57:45,791 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:57:46,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:50,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:52,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:56,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:57:57,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:57,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:57:58,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:01,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:58:01,013 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:58:01,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:01,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:02,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:03,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:03,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:04,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:05,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:05,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:06,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:08,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:08,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:09,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:10,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:10,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:12,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:13,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:13,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:14,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:15,802 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:58:16,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:58:16,236 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:58:16,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:17,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:17,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:19,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:20,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:22,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:22,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:24,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:24,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:25,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:25,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:26,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:27,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:27,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:28,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:29,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:29,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:30,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:58:31,571 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:58:31,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:31,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:32,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:32,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:34,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:35,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:35,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:36,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:38,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:40,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:40,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:40,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:41,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:42,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:42,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:43,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:44,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:45,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:45,872 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:58:46,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:46,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:58:46,812 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:58:48,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:48,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:49,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:49,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:50,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:52,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:53,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:53,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:54,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:55,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:55,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:56,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:58,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:58:59,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:58:59,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:01,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:02,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:59:02,024 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:59:03,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:05,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:07,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:09,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:10,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:10,598 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 15:59:10,796 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 15:59:10,874 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 15:59:11,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:11,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 15:59:12,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:15,986 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:59:17,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:59:17,253 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:59:17,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:21,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:25,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:29,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:32,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:59:32,553 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:59:33,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:35,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:39,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:41,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:45,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:46,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 15:59:47,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:47,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 15:59:47,916 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 15:59:48,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:48,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:51,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:53,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:55,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:57,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 15:59:57,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:57,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 15:59:58,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:00,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:01,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:02,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:02,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:00:03,283 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:00:03,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:03,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:05,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:07,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:08,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:08,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:09,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:11,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:13,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:13,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:13,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:15,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:15,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:00:16,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:16,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:17,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:18,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:18,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:00:18,505 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:00:19,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:20,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:20,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:21,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:22,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:22,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:23,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:24,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:24,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:25,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:27,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:29,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:29,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:30,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:30,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:31,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:31,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:32,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:33,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:00:33,729 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:00:33,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:34,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:34,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:35,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:36,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:36,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:37,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:38,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:38,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:39,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:41,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:41,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:42,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:42,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:43,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:43,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:45,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:46,185 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:00:47,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:47,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:47,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:49,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:00:49,044 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:00:49,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:51,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:53,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:55,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:57,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:00:57,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:00:59,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:03,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:04,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:01:04,266 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:01:07,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:11,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:15,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:16,264 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:01:19,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:01:19,485 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:01:19,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:22,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:26,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:30,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:32,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:34,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:01:34,853 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:01:34,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:38,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:40,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:44,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:44,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:46,339 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:01:46,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:47,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:47,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:48,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:49,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:49,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:50,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:01:50,145 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:01:50,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:51,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:51,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:52,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:53,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:53,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:56,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:01:57,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:57,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:01:58,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:00,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:00,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:00,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:02,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:02,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:03,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:04,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:05,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:02:05,369 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:02:06,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:06,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:07,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:07,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:08,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:09,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:09,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:10,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:11,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:11,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:12,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:13,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:13,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:14,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:16,417 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:02:16,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:18,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:18,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:20,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:02:20,596 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:02:20,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:20,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:21,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:21,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:22,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:24,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:24,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:25,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:25,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:26,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:28,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:28,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:29,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:29,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:30,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:31,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:31,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:32,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:33,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:33,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:34,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:35,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:02:35,872 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:02:36,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:38,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:39,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:39,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:40,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:42,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:44,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:02:44,714 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 16:02:44,958 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 16:02:44,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:45,028 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 16:02:45,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 16:02:46,532 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:02:46,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:50,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:51,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:02:51,097 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:02:56,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:02:58,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:02,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:06,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:03:06,326 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:03:06,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:10,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:12,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:16,606 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:03:16,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:18,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:21,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:03:21,550 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:03:22,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:25,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:27,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:31,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:33,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:35,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:36,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:36,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:03:36,880 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:03:37,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:38,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:38,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:40,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:41,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:43,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:44,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:44,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:45,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:45,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:46,678 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:03:47,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:47,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:48,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:48,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:50,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:50,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:52,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:03:52,108 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:03:52,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:52,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:54,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:56,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:57,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:57,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:03:58,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:58,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:03:59,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:00,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:00,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:01,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:02,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:02,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:03,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:04,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:04,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:05,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:06,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:06,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:07,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:04:07,362 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:04:08,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:09,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:09,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:10,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:12,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:12,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:12,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:13,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:14,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:14,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:15,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:16,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:16,753 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:04:18,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:18,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:18,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:20,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:20,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:21,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:22,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:22,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:04:22,638 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:04:23,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:23,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:24,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:26,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:28,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:28,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:28,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:30,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:32,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:32,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:32,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:32,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:32,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:32,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:32,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:32,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:32,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:32,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:32,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:32,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:32,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:04:34,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:37,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:04:37,868 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:04:40,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:44,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:46,830 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:04:48,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:52,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:53,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:04:53,096 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:04:56,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:04:58,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:02,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:04,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:08,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:08,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:05:08,322 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:05:10,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:14,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:16,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:16,907 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:05:18,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:22,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:23,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:05:23,544 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:05:24,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:26,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:28,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:28,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:28,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:29,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:30,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:31,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:31,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:34,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:35,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:35,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:36,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:36,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:37,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:38,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:05:38,820 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:05:38,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:38,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:39,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:39,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:39,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:39,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:39,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:39,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:39,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:39,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:39,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:39,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:40,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:41,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:41,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:42,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:43,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:43,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:44,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:45,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:45,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:46,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:46,984 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:05:48,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:48,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:48,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:49,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:50,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:50,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:51,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:52,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:52,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:53,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:54,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:05:54,120 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:05:54,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:54,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:55,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:56,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:56,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:05:58,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:58,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:05:59,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:00,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:00,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:02,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:03,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:03,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:04,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:04,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:05,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:06,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:07,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:07,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:08,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:09,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:06:09,342 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:06:10,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:10,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:10,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:11,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:12,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:14,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:16,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:17,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:06:17,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:17,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:18,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:18,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:19,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 16:06:19,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 16:06:19,353 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 16:06:19,425 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 16:06:20,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:20,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 16:06:22,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:24,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:24,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:06:24,624 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:06:26,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:28,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:30,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:33,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:35,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:37,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:39,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:39,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:06:39,930 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:06:41,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:43,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:45,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:47,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:47,493 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:06:49,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:51,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:53,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:55,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:55,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:06:55,205 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:06:57,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:06:59,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:01,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:03,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:05,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:07,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:09,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:10,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:07:10,512 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:07:11,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:13,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:15,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:17,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:17,570 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:07:19,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:21,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:23,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:25,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:25,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:07:25,734 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:07:27,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:29,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:31,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:33,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:35,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:37,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:39,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:41,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:07:41,088 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:07:41,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:43,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:45,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:47,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:47,646 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:07:50,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:52,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:54,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:56,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:07:56,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:07:56,314 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:07:58,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:00,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:02,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:04,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:06,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:08,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:10,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:11,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:08:11,694 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:08:12,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:14,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:16,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:17,723 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:08:18,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:20,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:22,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:24,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:26,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:26,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:08:26,914 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:08:28,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:30,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:32,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:34,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:36,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:38,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:40,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:42,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:08:42,191 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:08:42,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:44,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:46,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:47,796 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:08:48,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:50,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:52,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:54,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:56,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:08:57,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:08:57,452 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:08:58,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:01,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:03,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:05,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:07,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:09,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:11,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:12,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:09:12,672 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:09:13,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:15,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:17,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:17,873 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:09:19,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:21,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:23,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:25,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:27,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:28,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:09:28,041 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:09:29,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:31,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:33,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:35,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:37,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:39,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:41,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:43,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:43,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:09:43,261 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:09:45,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:47,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:47,948 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:09:49,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:51,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:53,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:55,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:57,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:09:58,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:09:58,562 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:09:59,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:01,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:03,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:05,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:07,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:10,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:12,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:13,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:10:13,823 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:10:14,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:16,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:18,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:10:18,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:20,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:22,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:24,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:26,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:28,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:29,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:10:29,109 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:10:30,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:32,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:34,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:36,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:38,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:40,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:42,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:44,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:44,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:10:44,334 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:10:46,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:48,097 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:10:48,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:50,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:52,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:54,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:56,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:58,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:10:59,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:10:59,628 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:11:00,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:02,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:04,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:06,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:08,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:10,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:12,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:14,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:14,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:11:14,849 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:11:16,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:18,166 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:11:18,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:20,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:22,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:24,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:26,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:28,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:30,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:11:30,181 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:11:30,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:32,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:34,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:37,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:39,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:41,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:43,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:45,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:45,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:11:45,468 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:11:47,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:48,242 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:11:49,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:51,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:53,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:55,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:57,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:11:59,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:00,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:12:00,764 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:12:01,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:03,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:05,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:07,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:09,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:11,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:13,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:15,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:15,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:12:15,992 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:12:17,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:18,317 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:12:19,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:21,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:23,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:25,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:27,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:29,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:31,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:12:31,254 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:12:31,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:33,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:35,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:37,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:39,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:41,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:43,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:45,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:46,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:12:46,549 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:12:47,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:48,394 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:12:49,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:51,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:53,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:55,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:12:58,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:00,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:01,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:13:01,767 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:13:02,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:04,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:06,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:08,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:10,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:12,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:14,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:16,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:16,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:13:16,980 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:13:18,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:18,470 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:13:20,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:22,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:24,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:26,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:28,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:30,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:32,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:13:32,317 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:13:32,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:34,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:36,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:38,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:40,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:42,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:44,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:46,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:47,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:13:47,540 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:13:48,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:48,543 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:13:50,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:52,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:54,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:56,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:13:58,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:00,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:02,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:02,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:14:02,832 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:14:04,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:06,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:08,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:10,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:12,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:14,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:16,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:18,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:14:18,085 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:14:18,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:18,619 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:14:20,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:22,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:25,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:27,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:29,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:31,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:33,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:14:33,309 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:14:33,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:35,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:37,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:39,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:41,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:43,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:45,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:47,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:48,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:14:48,661 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:14:48,884 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:14:49,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:51,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:53,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:55,355 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:57,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:14:59,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:01,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:03,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:03,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:15:03,983 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:15:05,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:07,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:09,361 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:11,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:13,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:15,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:17,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:18,769 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:15:19,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:15:19,203 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:15:19,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:21,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:23,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:25,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:27,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:29,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:31,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:33,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:34,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:15:34,495 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:15:35,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:37,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:39,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:41,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:43,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:45,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:47,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:48,845 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:15:49,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:49,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:15:49,749 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:15:51,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:53,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:56,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:15:58,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:00,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:02,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:04,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:05,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:16:05,043 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:16:06,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:08,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:10,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:12,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:14,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:16,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:18,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:18,922 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:16:20,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:16:20,376 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:16:20,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:22,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:24,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:26,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:28,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:30,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:32,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:34,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:35,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:16:35,629 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:16:36,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:38,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:40,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:42,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:44,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:46,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:48,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:48,993 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:16:50,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:50,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:16:50,855 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:16:52,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:54,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:56,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:16:58,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:00,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:02,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:04,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:06,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:17:06,113 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:17:07,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:09,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:11,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:13,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:15,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:17,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:19,062 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:17:19,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:21,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:17:21,343 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:17:21,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:23,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:25,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:27,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:29,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:31,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:33,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:35,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:36,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:17:36,587 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:17:37,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:39,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:41,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:43,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:45,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:47,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:49,136 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:17:49,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:51,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:51,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:17:51,803 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:17:53,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:55,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:57,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:17:59,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:01,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:03,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:05,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:07,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:18:07,086 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:18:07,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:09,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:11,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:14,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:16,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:18,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:19,212 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:18:20,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:22,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:18:22,310 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:18:22,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:24,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:26,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:28,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:30,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:32,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:34,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:36,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:37,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:18:37,647 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:18:38,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:40,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:42,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:44,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:46,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:48,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:49,290 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:18:50,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:52,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:52,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:18:52,967 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:18:54,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:56,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:18:58,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:00,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:02,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:04,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:06,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:08,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:19:08,191 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:19:08,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:10,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:12,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:14,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:16,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:18,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:19,368 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:19:20,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:22,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:23,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:19:23,473 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:19:24,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:26,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:28,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:30,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:32,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:34,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:36,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:38,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:38,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:19:38,720 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:19:40,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:42,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:44,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:46,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:48,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:49,442 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:19:50,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:52,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:53,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:19:53,937 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:19:54,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:56,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:19:58,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:01,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:03,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:05,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:07,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:09,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:20:09,226 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:20:09,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:11,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:13,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:15,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:17,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:19,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:19,519 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:20:21,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:23,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:24,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:20:24,593 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:20:25,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:27,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:29,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:31,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:33,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:35,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:37,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:39,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:39,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:20:39,884 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:20:41,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:43,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:45,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:47,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:49,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:49,594 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:20:51,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:53,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:55,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:20:55,116 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:20:55,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:57,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:20:59,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:01,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:03,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:05,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:07,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:09,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:10,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:21:10,413 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:21:11,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:13,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:15,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:17,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:19,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:19,670 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:21:21,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:24,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:25,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:21:25,630 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:21:26,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:28,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:30,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:32,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:34,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:36,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:38,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:40,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:40,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:21:40,946 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:21:42,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:44,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:46,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:48,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:49,744 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:21:50,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:52,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:54,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:56,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:21:56,313 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:21:56,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:21:58,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:00,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:02,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:04,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:06,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:08,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:10,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:11,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:22:11,536 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:22:12,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:14,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:16,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:18,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:19,812 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:22:20,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:22,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:24,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:26,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:26,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:22:26,869 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:22:28,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:30,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:33,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:35,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:37,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:39,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:41,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:42,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:22:42,279 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:22:43,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:45,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:47,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:49,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:49,883 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:22:51,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:53,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:55,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:57,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:22:57,503 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:22:57,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:22:59,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:01,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:03,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:05,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:07,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:09,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:11,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:12,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:23:12,831 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:23:13,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:15,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:17,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:19,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:19,958 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:23:21,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:23,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:25,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:27,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:28,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:23:28,056 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:23:29,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:31,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:33,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:35,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:37,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:39,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:41,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:43,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:23:43,298 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:23:43,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:45,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:47,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:49,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:50,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:23:51,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:53,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:55,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:57,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:23:58,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:23:58,614 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:23:59,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:01,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:03,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:05,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:07,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:09,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:11,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:13,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:13,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:24:13,837 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:24:15,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:17,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:19,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:20,107 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:24:21,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:23,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:25,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:27,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:29,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:24:29,118 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:24:29,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:31,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:34,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:36,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:38,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:40,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:42,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:24:44,485 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:24:44,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:46,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:48,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:50,182 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:24:50,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:52,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:54,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:56,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:58,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:24:59,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:24:59,705 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:25:00,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:02,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:04,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:06,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:08,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:10,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:12,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:14,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:15,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:25:15,055 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:25:16,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:18,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:20,254 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:25:20,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:22,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:24,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:26,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:28,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:30,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:25:30,277 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:25:30,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:32,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:34,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:36,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:38,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:40,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:42,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:44,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:45,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:25:45,556 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:25:46,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:48,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:50,328 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:25:50,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:52,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:54,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:57,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:25:59,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:00,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:26:00,947 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:26:01,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:03,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:05,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:07,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:09,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:11,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:13,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:15,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:16,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:26:16,199 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:26:17,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:19,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:20,401 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:26:21,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:23,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:25,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:27,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:29,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:31,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:26:31,424 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:26:31,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:33,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:35,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:37,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:39,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:41,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:43,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:45,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:46,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:26:46,645 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:26:47,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:49,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:50,478 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:26:51,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:53,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:55,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:57,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:26:59,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:01,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:02,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:27:02,018 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:27:03,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:05,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:07,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:10,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:12,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:14,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:16,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:17,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:27:17,365 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:27:18,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:20,554 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:27:20,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:22,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:24,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:26,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:28,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:30,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:32,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:27:32,617 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:27:32,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:34,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:36,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:38,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:40,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:42,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:44,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:46,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:47,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:27:47,849 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:27:48,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:50,631 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:27:50,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:52,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:54,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:56,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:27:58,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:00,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:02,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:03,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:28:03,118 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:28:04,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:06,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:08,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:10,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:12,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:14,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:16,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:18,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:28:18,519 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:28:18,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:20,701 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:28:20,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:22,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:25,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:27,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:29,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:31,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:33,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:33,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:28:33,814 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:28:35,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:37,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:39,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:41,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:43,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:45,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:47,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:49,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:28:49,036 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:28:49,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:50,777 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:28:51,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:53,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:55,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:57,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:28:59,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:01,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:03,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:04,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:29:04,332 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:29:05,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:07,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:09,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:11,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:13,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:15,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:17,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:19,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:29:19,604 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:29:19,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:20,851 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:29:21,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:23,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:25,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:27,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:29,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:31,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:33,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:34,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:29:34,856 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:29:35,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:37,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:39,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:41,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:43,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:45,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:47,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:49,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:50,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:29:50,082 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:29:50,925 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:29:51,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:53,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:55,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:57,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:29:59,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:01,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:03,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:05,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:30:05,355 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:30:05,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:07,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:09,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:11,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:13,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:15,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:17,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:19,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:20,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:30:20,569 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:30:20,998 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:30:22,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:24,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:26,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:28,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:30,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:32,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:34,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:35,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:30:35,793 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:30:36,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:38,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:40,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:42,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:44,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:46,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:48,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:50,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:51,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:30:51,069 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:30:51,310 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:30:52,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:54,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:56,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:30:58,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:00,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:02,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:04,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:06,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:31:06,433 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:31:06,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:08,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:10,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:12,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:14,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:16,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:18,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:20,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:21,146 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:31:21,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:31:21,724 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:31:22,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:24,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:26,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:28,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:30,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:32,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:34,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:36,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:37,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:31:37,136 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:31:39,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:41,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:43,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:45,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:47,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:49,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:51,222 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:31:51,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:52,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:31:52,433 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:31:53,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:55,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:57,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:31:59,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:01,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:03,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:05,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:07,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:32:07,781 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:32:07,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:09,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:11,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:13,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:15,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:17,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:19,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:21,293 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:32:21,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:23,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:32:23,052 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:32:23,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:25,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:27,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:29,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:31,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:33,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:35,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:37,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:38,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:32:38,384 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:32:39,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:41,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:43,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:45,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:47,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:49,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:51,363 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:32:51,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:53,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:32:53,648 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:32:53,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:55,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:57,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:32:59,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:01,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:03,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:05,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:07,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:08,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:33:08,878 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:33:09,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:11,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:14,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:16,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:18,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:20,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:21,436 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:33:22,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:24,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:33:24,282 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:33:24,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:26,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:28,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:30,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:32,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:34,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:36,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:38,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:39,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:33:39,504 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:33:40,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:42,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:44,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:46,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:48,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:50,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:51,509 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:33:52,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:54,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:33:54,799 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:33:54,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:56,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:33:58,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:00,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:02,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:04,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:06,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:08,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:10,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:34:10,026 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:34:10,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:12,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:14,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:16,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:18,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:20,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:21,583 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:34:22,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:24,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:25,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:34:25,400 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:34:26,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:29,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:31,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:33,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:35,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:37,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:39,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:40,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:34:40,677 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:34:41,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:43,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:45,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:47,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:49,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:51,658 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:34:51,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:53,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:55,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:55,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:34:55,928 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:34:57,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:34:59,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:01,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:03,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:05,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:07,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:09,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:11,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:35:11,153 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:35:11,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:13,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:15,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:17,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:19,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:21,725 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:35:21,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:23,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:25,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:26,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:35:26,457 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:35:27,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:29,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:31,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:33,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:35,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:37,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:39,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:41,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:35:41,825 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:35:41,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:43,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:45,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:47,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:50,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:51,799 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:35:52,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:54,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:56,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:35:57,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:35:57,047 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:35:58,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:00,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:02,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:04,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:06,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:08,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:10,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:12,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:36:12,270 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:36:12,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:14,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:16,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:18,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:20,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:21,875 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:36:22,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:24,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:26,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:27,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:36:27,494 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:36:28,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:30,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:32,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:34,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:36,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:38,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:40,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:42,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:36:42,803 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:36:42,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:44,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:46,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:48,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:50,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:51,947 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:36:52,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:54,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:56,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:36:58,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:36:58,204 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:36:58,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:00,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:02,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:04,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:07,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:09,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:11,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:13,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:37:13,440 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:37:13,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:15,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:17,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:19,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:21,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:22,019 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:37:23,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:25,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:27,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:28,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:37:28,700 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:37:29,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:31,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:33,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:35,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:37,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:39,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:41,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:43,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:44,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:37:44,089 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:37:45,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:47,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:49,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:51,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:52,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:37:53,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:55,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:57,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:37:59,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:37:59,309 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:37:59,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:01,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:03,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:05,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:07,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:09,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:11,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:13,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:14,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:38:14,649 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:38:15,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:17,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:20,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:22,166 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:38:22,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:24,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:26,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:28,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:29,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:38:29,888 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:38:30,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:32,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:34,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:36,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:38,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:40,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:42,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:44,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:45,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:38:45,225 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:38:46,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:48,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:50,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:52,238 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:38:52,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:54,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:56,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:38:58,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:00,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:39:00,452 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:39:00,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:02,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:04,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:07,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:09,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:11,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:13,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:15,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:15,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:39:15,674 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:39:17,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:19,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:21,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:22,316 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:39:23,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:25,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:27,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:29,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:30,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:39:30,894 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:39:31,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:33,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:35,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:37,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:39,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:41,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:43,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:45,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:46,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:39:46,184 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:39:47,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:49,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:51,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:52,393 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:39:53,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:55,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:57,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:39:59,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:01,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:01,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:40:01,405 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:40:03,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:06,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:08,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:10,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:12,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:14,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:16,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:16,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:40:16,625 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:40:18,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:20,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:22,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:22,468 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:40:24,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:26,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:28,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:30,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:31,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:40:31,843 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:40:32,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:34,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:36,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:38,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:40,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:42,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:44,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:46,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:47,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:40:47,216 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:40:48,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:50,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:52,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:52,544 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:40:54,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:56,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:40:58,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:00,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:02,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:02,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:41:02,435 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:41:04,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:06,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:08,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:10,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:12,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:14,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:16,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:17,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:41:17,721 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:41:18,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:20,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:22,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:22,619 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:41:24,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:26,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:28,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:30,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:32,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:41:32,933 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:41:33,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:35,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:37,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:39,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:41,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:43,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:45,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:47,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:48,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:41:48,212 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:41:49,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:51,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:52,695 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:41:53,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:55,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:57,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:41:59,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:01,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:03,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:03,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:42:03,498 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:42:05,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:07,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:09,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:11,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:13,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:15,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:17,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:18,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:42:18,726 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:42:19,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:21,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:22,768 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:42:23,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:25,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:27,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:29,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:31,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:33,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:33,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:42:33,979 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:42:35,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:37,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:39,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:41,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:43,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:45,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:47,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:49,100 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:49,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:42:49,203 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:42:51,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:52,843 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:42:53,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:56,103 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:42:58,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:00,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:02,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:04,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:04,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:43:04,427 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:43:06,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:08,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:10,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:12,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:14,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:16,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:18,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:19,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:43:19,750 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:43:20,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:22,112 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:22,916 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:43:24,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:26,114 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:28,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:30,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:32,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:34,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:35,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:43:35,216 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:43:36,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:38,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:40,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:42,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:44,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:46,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:48,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:50,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:50,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:43:50,440 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:43:52,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:52,995 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:43:54,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:56,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:43:58,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:00,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:02,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:04,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:05,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:44:05,662 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:44:06,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:08,132 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:10,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:12,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:14,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:16,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:18,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:20,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:21,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:44:21,057 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:44:22,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:23,070 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:44:24,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:26,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:28,140 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:31,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:33,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:35,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:36,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:44:36,334 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:44:37,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:39,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:41,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:43,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:45,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:47,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:49,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:51,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:51,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:44:51,636 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:44:53,146 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:44:53,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:55,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:57,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:44:59,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:01,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:03,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:05,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:06,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:45:06,925 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:45:07,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:09,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:11,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:13,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:15,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:17,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:19,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:21,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:22,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:45:22,193 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:45:23,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:23,220 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:45:25,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:27,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:29,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:31,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:33,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:35,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:37,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:37,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:45:37,489 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:45:39,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:41,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:43,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:45,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:47,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:49,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:51,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:52,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:45:52,778 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:45:53,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:53,297 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:45:55,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:45:58,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:00,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:02,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:04,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:06,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:07,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:46:07,996 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:46:08,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:10,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:12,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:14,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:16,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:18,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:20,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:22,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:23,374 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:46:23,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:46:23,401 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:46:24,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:26,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:28,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:30,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:32,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:34,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:36,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:38,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:38,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:46:38,860 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:46:40,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:42,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:44,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:46,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:48,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:50,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:52,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:53,449 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:46:54,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:46:54,169 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:46:54,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:56,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:46:58,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:00,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:02,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:04,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:06,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:08,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:09,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:47:09,395 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:47:10,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:12,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:14,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:16,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:18,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:20,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:22,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:23,526 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:47:24,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:24,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:47:24,680 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:47:26,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:28,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:31,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:33,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:35,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:37,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:39,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:39,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:47:39,909 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:47:41,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:43,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:45,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:47,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:49,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:51,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:53,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:53,600 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:47:55,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:47:55,129 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:47:55,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:57,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:47:59,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:01,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:03,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:05,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:07,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:09,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:10,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:48:10,467 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:48:11,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:13,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:15,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:17,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:19,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:21,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:23,673 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:48:24,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:25,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:48:25,768 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:48:26,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:28,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:30,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:32,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:34,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:36,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:38,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:40,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:40,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:48:40,988 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:48:42,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:44,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:46,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:48,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:50,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:52,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:53,746 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:48:54,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:56,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:48:56,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:48:56,299 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:48:58,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:00,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:02,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:04,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:06,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:08,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:10,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:11,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:49:11,545 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:49:12,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:14,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:16,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:18,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:20,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:22,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:23,818 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:49:24,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:26,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:26,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:49:26,766 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:49:28,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:30,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:32,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:34,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:36,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:38,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:40,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:41,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:49:41,989 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:49:42,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:44,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:46,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:48,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:50,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:52,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:53,895 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:49:55,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:57,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:49:57,266 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:49:57,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:49:59,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:01,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:03,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:05,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:07,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:09,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:11,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:12,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:50:12,493 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:50:13,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:15,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:17,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:19,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:21,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:23,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:23,971 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:50:25,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:27,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:27,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:50:27,715 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:50:29,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:31,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:33,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:35,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:37,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:39,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:41,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:43,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:50:43,027 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:50:43,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:45,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:47,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:49,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:51,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:53,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:54,048 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:50:55,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:57,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:50:58,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:50:58,267 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:50:59,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:01,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:03,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:05,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:07,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:09,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:11,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:13,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:13,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:51:13,561 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:51:15,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:17,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:19,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:21,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:23,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:24,121 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:51:25,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:27,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:28,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:51:28,851 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:51:29,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:31,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:33,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:35,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:37,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:39,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:41,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:44,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:51:44,238 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:51:44,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:46,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:48,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:50,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:52,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:54,196 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:51:54,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:56,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:58,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:51:59,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:51:59,500 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:52:00,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:02,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:04,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:06,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:08,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:10,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:12,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:14,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:14,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:52:14,792 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:52:16,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:18,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:20,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:22,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:24,272 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:52:24,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:26,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:28,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:30,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:52:30,230 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:52:30,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:32,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:34,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:36,355 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:38,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:40,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:42,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:44,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:45,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:52:45,498 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:52:46,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:49,361 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:51,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:53,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:54,347 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:52:55,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:57,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:52:59,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:00,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:53:00,799 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:53:01,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:03,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:05,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:07,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:09,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:11,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:13,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:15,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:16,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:53:16,051 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:53:17,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:19,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:21,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:23,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:24,419 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:53:25,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:27,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:29,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:31,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:53:31,360 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:53:31,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:33,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:35,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:37,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:39,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:41,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:43,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:45,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:46,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:53:46,579 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:53:47,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:49,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:51,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:53,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:54,494 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:53:55,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:57,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:53:59,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:01,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:01,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:54:01,902 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:54:03,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:05,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:07,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:09,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:11,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:13,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:15,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:17,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:54:17,126 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:54:17,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:20,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:22,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:24,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:24,570 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:54:26,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:28,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:30,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:32,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:33,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:54:33,344 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:54:34,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:36,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:38,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:40,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:42,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:44,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:46,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:48,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:48,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:54:48,564 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:54:50,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:52,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:54,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:54,642 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:54:56,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:54:58,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:00,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:02,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:03,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:55:03,902 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:55:04,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:06,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:08,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:10,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:12,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:14,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:16,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:18,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:19,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:55:19,202 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:55:20,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:22,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:24,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:24,710 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:55:26,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:28,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:30,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:32,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:34,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:55:34,415 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:55:34,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:36,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:38,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:40,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:42,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:44,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:46,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:48,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:49,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:55:49,774 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:55:50,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:52,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:54,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:54,781 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:55:56,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:55:58,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:00,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:02,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:04,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:05,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:56:05,174 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:56:06,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:08,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:10,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:12,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:14,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:16,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:18,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:20,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:56:20,394 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:56:20,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:22,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:24,858 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:56:25,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:27,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:29,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:31,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:33,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:35,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:35,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:56:35,670 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:56:37,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:39,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:41,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:43,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:45,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:47,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:49,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:50,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:56:50,944 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:56:51,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:53,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:54,936 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:56:55,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:57,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:56:59,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:01,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:03,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:05,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:06,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:57:06,165 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:57:07,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:09,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:11,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:13,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:15,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:17,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:19,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:21,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:21,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:57:21,536 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:57:23,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:25,010 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:57:26,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:28,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:30,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:32,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:34,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:36,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:36,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:57:36,848 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:57:38,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:40,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:42,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:44,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:46,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:48,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:50,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:52,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:57:52,115 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:57:52,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:54,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:55,086 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:57:56,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:57:58,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:00,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:02,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:04,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:06,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:07,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:58:07,487 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:58:08,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:10,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:12,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:14,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:16,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:18,506 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:20,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:22,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:22,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:58:22,788 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:58:24,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:25,161 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:58:26,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:28,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:30,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:32,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:34,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:36,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:38,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:58:38,097 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:58:38,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:40,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:43,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:45,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:47,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:49,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:51,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:53,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:58:53,356 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:58:53,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:55,239 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:58:55,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:57,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:58:59,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:01,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:03,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:05,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:07,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:08,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:59:08,690 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:59:09,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:11,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:13,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:15,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:17,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:19,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:21,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:23,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:23,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:59:23,986 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:59:25,323 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:59:25,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:27,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:29,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:31,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:33,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:35,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:37,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:39,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:59:39,294 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:59:39,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:41,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:43,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:45,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:48,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:50,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:52,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:54,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 16:59:54,521 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 16:59:54,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:55,374 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 16:59:56,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 16:59:58,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:00,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:02,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:04,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:06,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:08,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:09,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:00:09,881 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:00:10,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:12,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:14,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:16,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:18,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:20,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:22,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:24,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:25,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:00:25,206 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:00:25,440 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:00:26,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:28,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:30,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:32,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:34,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:36,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:38,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:40,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:00:40,488 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:00:40,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:42,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:44,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:46,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:48,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:50,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:52,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:54,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:55,507 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:00:55,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:00:55,716 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:00:56,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:00:58,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:00,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:03,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:05,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:07,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:09,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:11,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:01:11,109 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:01:11,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:13,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:15,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:17,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:19,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:21,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:23,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:25,572 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:01:25,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:26,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:01:26,335 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:01:27,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:29,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:31,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:33,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:35,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:37,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:39,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:41,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:41,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:01:41,714 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:01:43,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:45,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:47,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:49,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:51,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:53,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:55,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:55,640 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:01:57,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:01:57,048 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:01:57,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:01:59,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:01,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:03,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:05,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:07,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:09,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:11,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:12,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:02:12,304 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:02:13,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:15,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:17,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:19,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:21,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:24,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:25,709 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:02:26,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:27,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:02:27,654 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:02:28,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:30,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:32,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:34,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:36,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:38,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:40,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:42,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:42,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:02:42,958 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:02:44,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:46,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:48,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:50,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:52,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:54,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:55,778 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:02:56,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:02:58,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:02:58,265 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:02:58,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:00,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:02,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:04,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:06,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:08,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:10,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:12,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:13,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:03:13,571 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:03:14,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:16,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:18,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:20,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:22,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:24,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:25,848 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:03:26,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:28,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:28,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:03:28,839 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:03:30,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:32,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:34,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:37,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:39,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:41,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:43,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:44,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:03:44,135 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:03:45,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:47,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:49,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:51,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:53,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:55,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:55,913 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:03:57,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:03:59,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:03:59,513 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:03:59,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:01,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:03,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:05,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:07,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:09,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:11,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:13,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:14,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:04:14,811 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:04:15,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:17,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:19,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:21,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:23,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:25,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:25,978 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:04:27,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:29,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:30,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:04:30,125 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:04:31,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:33,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:35,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:37,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:39,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:41,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:43,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:45,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:04:45,403 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:04:45,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:47,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:49,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:51,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:53,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:55,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:56,044 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:04:57,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:04:59,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:00,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:05:00,698 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:05:01,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:03,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:05,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:07,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:09,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:11,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:13,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:15,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:15,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:05:15,998 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:05:17,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:19,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:21,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:23,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:25,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:26,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:05:27,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:29,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:31,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:05:31,220 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:05:32,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:34,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:36,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:38,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:40,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:42,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:44,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:46,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:05:46,630 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:05:46,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:48,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:50,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:52,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:54,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:56,189 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:05:56,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:05:58,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:00,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:01,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:06:01,933 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:06:02,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:04,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:06,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:08,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:10,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:12,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:14,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:16,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:17,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:06:17,246 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:06:18,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:20,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:22,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:24,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:26,257 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:06:26,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:28,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:30,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:32,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:06:32,509 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:06:32,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:34,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:36,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:38,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:40,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:42,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:45,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:47,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:47,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:06:47,734 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:06:49,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:51,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:53,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:55,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:56,322 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:06:57,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:06:59,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:01,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:02,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:07:02,953 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:07:03,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:05,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:07,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:09,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:11,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:13,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:15,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:17,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:18,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:07:18,249 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:07:19,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:21,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:23,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:25,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:26,390 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:07:27,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:29,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:31,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:33,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:07:33,503 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:07:33,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:35,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:37,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:39,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:41,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:43,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:45,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:47,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:48,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:07:48,770 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:07:49,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:51,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:53,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:55,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:56,459 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:07:57,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:07:59,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:01,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:03,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:03,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:08:03,992 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:08:05,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:07,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:09,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:11,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:13,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:15,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:17,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:19,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:08:19,207 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:08:19,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:21,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:23,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:26,531 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:08:26,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:28,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:30,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:32,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:34,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:08:34,529 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:08:34,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:36,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:38,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:40,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:42,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:44,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:46,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:48,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:49,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:08:49,744 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:08:50,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:52,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:54,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:56,599 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:08:56,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:08:58,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:00,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:02,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:04,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:04,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:09:04,964 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:09:06,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:08,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:10,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:12,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:14,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:16,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:18,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:20,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:09:20,257 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:09:20,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:22,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:24,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:26,669 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:09:26,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:28,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:30,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:32,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:34,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:35,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:09:35,587 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:09:36,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:38,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:40,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:43,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:45,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:47,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:49,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:50,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:09:50,810 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:09:51,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:53,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:55,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:56,747 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:09:57,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:09:59,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:01,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:03,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:05,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:06,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:10:06,089 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:10:07,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:09,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:11,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:13,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:15,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:17,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:19,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:21,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:10:21,445 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:10:21,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:23,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:25,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:26,821 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:10:27,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:29,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:31,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:33,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:35,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:36,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:10:36,705 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:10:37,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:39,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:41,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:43,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:45,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:47,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:49,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:51,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:52,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:10:52,400 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:10:53,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:55,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:56,898 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:10:57,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:10:59,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:01,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:04,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:06,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:07,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:11:07,621 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:11:08,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:10,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:12,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:14,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:16,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:18,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:20,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:22,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:22,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:11:22,841 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:11:24,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:26,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:26,973 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:11:28,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:30,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:32,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:34,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:36,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:38,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:11:38,197 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:11:38,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:40,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:42,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:44,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:46,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:48,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:50,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:52,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:53,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:11:53,452 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:11:54,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:56,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:11:57,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:11:58,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:00,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:02,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:04,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:06,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:08,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:12:08,674 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:12:08,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:10,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:12,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:14,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:16,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:18,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:20,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:22,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:23,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:12:23,994 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:12:24,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:26,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:27,127 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:12:28,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:30,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:32,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:34,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:36,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:38,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:39,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:12:39,241 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:12:40,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:42,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:45,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:47,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:49,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:51,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:53,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:54,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:12:54,460 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:12:55,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:57,206 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:12:57,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:12:59,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:01,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:03,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:05,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:07,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:09,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:13:09,771 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:13:09,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:11,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:13,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:15,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:17,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:19,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:21,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:23,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:24,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:13:24,994 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:13:25,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:27,283 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:13:27,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:29,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:31,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:33,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:35,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:37,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:39,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:40,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:13:40,217 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:13:41,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:43,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:45,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:47,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:49,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:51,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:53,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:55,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:13:55,558 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:13:55,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:57,357 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:13:57,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:13:59,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:01,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:03,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:05,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:07,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:09,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:10,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:14:10,887 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:14:11,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:13,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:15,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:17,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:19,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:21,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:24,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:26,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:14:26,159 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:14:26,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:27,434 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:14:28,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:30,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:32,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:34,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:36,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:38,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:40,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:41,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:14:41,450 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:14:42,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:44,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:46,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:48,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:50,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:52,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:54,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:56,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:14:56,717 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:14:56,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:14:57,511 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:14:58,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:00,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:02,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:04,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:06,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:08,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:10,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:11,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:15:11,945 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:15:12,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:14,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:16,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:18,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:20,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:22,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:24,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:26,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:27,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:15:27,295 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:15:27,587 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:15:28,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:30,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:32,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:35,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:37,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:39,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:41,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:42,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:15:42,550 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:15:43,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:45,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:47,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:49,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:51,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:53,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:55,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:57,660 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:15:57,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:15:57,880 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:15:57,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:15:59,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:01,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:03,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:05,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:07,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:09,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:11,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:13,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:16:13,101 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:16:13,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:15,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:17,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:19,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:21,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:23,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:25,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:27,736 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:16:27,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:28,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:16:28,338 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:16:29,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:16:43,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:16:43,558 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:16:57,816 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:16:58,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:16:58,800 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:17:14,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:17:14,068 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:17:27,891 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:17:29,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:17:29,335 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:17:44,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:17:44,603 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:17:45,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:17:45,978 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 17:17:46,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 17:17:46,107 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 17:17:47,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 17:17:48,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:17:54,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:17:57,974 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:17:59,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:17:59,852 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:18:10,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:18:15,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:18:15,076 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:18:28,052 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:18:30,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:18:30,307 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:18:45,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:18:45,531 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:18:58,127 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:19:00,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:19:00,755 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:19:15,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:19:15,981 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:19:28,202 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:19:31,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:19:31,211 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:19:46,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:19:46,451 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:19:58,276 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:20:01,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:20:01,678 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:20:16,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:20:16,904 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:20:28,353 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:20:32,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:20:32,135 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:20:47,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:20:47,357 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:20:58,432 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:21:02,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:21:02,584 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:21:17,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:21:17,809 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:21:28,511 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:21:33,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:21:33,035 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:21:48,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:21:48,272 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:21:58,589 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:22:00,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:03,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:22:03,496 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:22:04,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:08,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:12,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:16,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:18,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:22:18,727 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:22:20,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:24,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:26,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:28,665 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:22:30,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:32,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:34,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:22:34,041 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:22:37,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:39,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:43,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:45,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:47,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:49,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:22:49,271 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:22:51,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:53,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:55,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:57,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:22:58,735 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:22:59,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:01,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:01,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:03,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:03,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:04,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:23:04,499 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:23:05,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:05,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:05,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:07,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:09,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:10,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:10,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:11,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:12,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:12,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:13,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:15,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:16,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:16,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:17,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:18,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:18,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:19,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:19,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:23:19,729 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:23:21,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:21,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:21,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:22,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:23,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:23,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:24,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:25,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:25,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:26,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:27,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:28,806 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:23:29,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:29,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:29,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:31,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:31,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:32,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:33,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:34,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:23:34,997 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:23:35,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:35,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:36,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:36,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:37,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:37,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:38,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:39,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:41,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:41,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:41,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:42,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:43,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:44,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:44,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:45,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:45,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:47,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:48,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:48,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:48,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:48,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:48,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:48,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:49,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:49,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:49,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:49,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:49,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:49,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:49,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:50,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:23:50,222 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:23:51,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:52,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:52,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:23:55,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:23:58,875 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:23:59,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:03,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:05,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:24:05,444 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:24:07,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:11,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:15,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:17,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:20,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:24:20,756 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:24:21,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:23,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:27,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:28,944 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:24:29,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:31,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:35,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:36,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:24:36,124 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:24:37,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:39,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:43,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:45,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:48,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:50,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:51,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:24:51,413 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:24:52,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:54,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:54,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:54,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:57,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:24:58,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:24:59,017 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:25:00,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:00,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:00,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:01,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:01,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:01,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:01,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:01,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:01,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:01,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:01,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:01,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:01,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:02,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:04,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:04,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:06,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:06,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:25:06,705 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:25:07,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:07,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:08,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:10,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:11,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:11,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:12,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:12,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:13,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:14,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:14,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:15,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:16,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:16,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:18,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:20,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:20,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:21,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:21,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:25:21,973 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:25:22,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:22,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:24,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:24,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:25,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:25,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:26,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:26,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:27,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:28,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:29,088 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:25:30,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:31,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:31,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:32,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:33,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:33,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:34,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:35,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:35,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:36,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:36,470 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 17:25:36,708 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 17:25:36,775 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 17:25:37,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:25:37,187 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:25:37,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:37,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 17:25:38,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:44,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:48,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:52,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:52,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:25:52,400 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:25:54,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:54,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:54,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:25:58,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:25:59,175 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:26:02,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:04,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:07,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:26:07,712 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:26:08,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:10,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:14,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:16,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:18,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:22,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:22,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:26:22,974 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:26:24,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:26,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:29,244 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:26:30,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:32,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:34,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:36,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:38,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:26:38,244 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:26:38,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:40,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:42,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:42,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:44,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:44,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:45,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:46,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:47,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:47,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:48,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:48,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:49,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:50,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:51,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:51,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:52,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:53,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:26:53,485 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:26:54,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:55,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:55,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:56,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:58,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:26:59,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:26:59,311 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:27:00,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:02,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:02,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:03,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:03,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:04,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:05,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:05,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:06,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:06,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:07,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:08,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:08,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:27:08,711 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:27:09,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:09,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:10,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:12,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:12,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:13,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:13,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:14,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:14,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:15,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:16,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:16,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:17,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:18,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:19,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:19,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:20,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:21,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:21,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:22,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:23,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:23,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:23,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:23,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:23,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:23,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:23,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:23,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:23,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:23,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:23,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:23,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:24,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:27:24,079 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:27:25,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:29,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:29,374 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:27:31,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:31,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:35,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:35,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:35,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:39,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:39,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:27:39,336 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:27:43,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:47,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:51,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:51,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:51,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:27:53,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:54,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:27:54,558 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:27:57,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:59,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:27:59,442 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:28:03,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:05,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:09,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:09,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:28:09,787 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:28:11,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:15,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:17,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:19,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:21,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:25,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:28:25,151 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:28:25,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:27,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:29,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:29,514 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:28:31,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:33,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:35,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:36,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:36,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:39,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:39,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:28:40,528 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:28:40,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:40,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:41,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:43,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:45,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:45,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:45,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:46,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:47,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:49,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:49,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:51,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:51,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:51,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:53,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:53,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:55,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:55,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:28:55,762 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:28:56,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:56,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:57,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:57,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:58,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:28:59,582 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:28:59,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:28:59,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:00,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:01,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:01,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:02,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:03,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:04,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:04,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:05,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:05,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:06,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:07,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:07,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:08,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:09,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:09,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:29:10,983 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:29:10,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:10,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:11,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:12,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:12,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:13,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:13,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:14,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:15,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:15,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:15,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:15,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:15,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:15,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:15,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:15,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:15,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:15,340 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 17:29:15,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:15,554 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 17:29:15,617 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 17:29:16,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 17:29:17,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:21,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:25,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:26,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:29:26,269 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:29:27,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:27,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:29,671 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:29:30,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:33,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:33,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:34,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:34,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:38,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:40,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:41,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:29:41,490 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:29:44,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:46,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:49,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:49,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:29:50,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:52,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:56,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:56,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:29:56,720 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:29:58,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:29:59,742 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:30:02,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:04,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:06,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:08,506 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:11,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:30:11,950 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:30:12,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:14,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:16,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:18,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:20,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:22,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:24,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:26,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:26,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:27,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:30:27,172 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:30:28,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:29,813 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:30:30,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:31,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:31,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:32,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:32,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:34,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:35,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:35,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:36,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:36,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:37,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:38,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:40,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:41,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:41,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:42,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:42,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:30:42,983 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:30:44,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:44,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:44,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:45,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:46,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:46,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:46,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:46,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:46,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:46,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:46,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:46,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:46,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:46,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:47,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:48,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:48,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:49,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:50,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:50,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:52,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:52,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:53,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:54,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:55,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:55,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:56,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:56,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:57,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:30:58,237 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:30:58,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:58,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:30:59,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,885 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:30:59,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:30:59,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:00,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:01,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:01,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:02,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:04,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:04,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:04,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:07,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:08,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:09,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:09,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:12,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:13,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:31:13,462 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:31:16,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:20,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:22,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:22,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:24,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:28,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:28,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:31:28,685 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:31:29,956 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:31:30,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:30,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:34,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:36,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:40,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:42,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:43,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:31:43,912 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:31:45,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:45,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:31:46,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:49,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:51,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:55,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:57,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:31:59,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:31:59,134 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:31:59,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:00,026 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:32:01,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:03,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:07,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:09,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:11,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:13,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:14,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:32:14,425 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:32:15,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:17,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:19,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:19,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:19,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:21,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:21,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:23,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:23,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:25,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:25,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:26,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:27,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:28,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:28,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:29,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:29,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:32:29,652 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:32:30,100 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:32:30,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:30,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:31,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:33,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:35,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:35,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:35,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:36,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:37,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:37,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:38,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:39,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:40,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:40,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:41,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:42,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:42,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:43,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:43,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:44,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:32:44,882 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:32:45,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:45,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:45,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:46,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:46,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:46,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:46,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:46,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:46,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:47,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:47,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:48,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:49,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:49,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:50,732 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 17:32:50,972 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 17:32:51,039 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 17:32:51,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:51,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 17:32:52,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:57,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:57,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:32:59,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:32:59,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:00,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:33:00,147 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:33:00,371 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:33:01,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:02,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:02,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:05,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:05,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:09,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:13,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:15,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:33:15,475 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:33:17,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:17,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:17,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:19,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:23,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:24,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:24,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:25,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:29,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:30,275 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:33:30,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:33:30,730 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:33:31,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:35,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:37,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:39,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:41,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:41,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:33:43,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:45,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:46,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:33:46,029 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:33:49,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:51,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:53,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:55,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:33:57,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:00,342 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:34:00,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:01,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:34:01,245 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:34:02,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:04,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:06,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:08,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:10,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:12,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:14,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:15,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:15,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:16,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:34:16,490 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:34:16,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:17,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:17,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:18,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:18,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:19,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:20,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:21,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:21,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:22,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:23,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:23,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:24,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:26,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:27,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:27,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:28,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:28,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:29,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,415 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:34:30,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:30,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:30,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:34:31,753 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:34:31,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:31,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:32,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:32,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:33,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:34,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:35,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:35,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:36,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:37,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:37,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:38,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:39,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:39,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:40,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:42,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:42,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:45,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:45,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:46,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:34:46,978 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:34:47,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:47,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:48,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:51,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:51,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:52,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:53,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:53,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:56,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:57,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:57,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:34:58,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:34:58,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:00,491 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:35:02,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:35:02,212 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:35:02,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:06,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:08,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:11,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:11,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:12,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:14,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:17,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:35:17,435 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:35:18,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:18,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:20,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:20,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:22,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:26,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:28,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:30,567 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:35:30,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:32,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:32,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:35:32,774 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:35:34,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:35,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:35,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:35:38,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:40,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:42,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:44,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:46,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:48,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:35:48,043 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:35:48,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:50,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:52,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:54,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:56,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:35:58,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:00,629 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:36:00,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:02,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:03,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:36:03,372 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:36:04,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:05,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:05,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:06,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:07,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:07,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:08,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:09,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:09,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:10,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:10,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:11,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:12,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:14,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:15,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:15,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:16,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:16,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:17,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:18,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:36:18,619 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:36:18,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:19,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:19,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:20,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:21,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:21,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:22,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:23,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:23,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:24,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:25,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:25,744 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 17:36:25,969 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 17:36:26,030 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 17:36:26,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:26,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 17:36:30,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:30,741 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:36:32,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:34,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:36:34,618 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:36:34,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:34,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:35,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:36,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:38,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:38,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:40,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:42,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:42,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:44,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:46,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,277 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:46,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:49,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:49,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:36:49,843 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:36:51,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:51,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:52,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:52,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:55,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:57,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:36:58,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:58,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:36:59,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:00,816 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:37:01,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:03,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:05,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:37:05,169 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:37:07,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:07,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:09,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:13,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:15,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:17,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:19,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:19,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:20,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:37:20,460 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:37:23,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:25,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:27,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:28,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:28,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:29,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:30,893 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:37:31,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:33,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:35,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:37:35,711 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:37:35,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:39,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:41,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:43,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:45,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:47,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:49,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:50,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:37:50,939 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:37:51,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:53,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:55,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:56,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:56,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:37:57,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:57,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:58,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:37:59,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:00,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:00,970 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:38:01,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:02,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:02,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:03,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:05,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:06,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:38:06,230 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:38:06,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:06,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:07,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:09,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:09,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:10,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:10,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:11,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:12,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:12,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:13,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:14,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:14,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:14,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:14,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:14,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:14,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:14,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:14,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:14,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:14,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:14,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:14,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:15,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:16,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:16,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:17,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:19,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:38:21,450 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:38:21,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:21,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:21,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:25,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:28,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:28,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:29,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:30,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:30,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:31,036 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:38:32,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:32,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:33,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:34,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:34,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:35,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:36,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:36,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:38:36,671 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:38:37,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:37,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:39,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:43,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:43,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:43,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:45,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:45,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:48,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:49,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:51,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,406 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,407 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:51,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:52,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:38:52,099 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:38:53,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:57,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:57,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:38:59,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:38:59,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:01,110 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:39:01,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:06,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:07,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:39:07,352 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:39:08,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:10,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:11,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:11,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:12,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:14,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:16,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:20,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:21,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:21,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:22,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:39:22,663 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:39:22,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:24,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:26,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:28,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:30,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:31,186 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:39:32,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:34,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:36,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:37,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:39:37,884 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:39:38,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:40,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:42,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:44,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:44,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:46,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:46,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:48,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:48,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:49,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:49,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:50,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:51,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:51,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:52,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:53,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:39:53,120 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:39:54,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:55,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:55,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:56,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:57,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:57,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:58,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:39:59,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:39:59,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 17:40:00,763 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 17:40:00,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:00,987 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 17:40:01,061 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 17:40:01,287 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:40:01,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:01,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 17:40:02,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:04,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:06,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:08,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:40:08,353 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:40:08,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:10,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:12,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:14,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:16,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:18,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:20,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:22,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:23,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:40:23,577 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:40:24,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:26,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:28,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:30,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:31,357 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:40:32,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:34,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:36,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:38,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:40:38,866 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:40:38,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:40,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:42,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:44,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:46,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:49,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:51,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:53,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:54,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:40:54,093 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:40:55,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:57,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:40:59,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:01,430 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:41:01,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:03,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:05,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:07,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:09,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:41:09,313 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:41:09,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:11,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:13,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:15,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:17,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:19,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:21,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:23,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:24,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:41:24,654 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:41:25,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:27,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:29,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:31,507 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:41:31,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:33,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:35,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:37,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:39,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:41:39,903 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:41:39,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:41,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:43,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:45,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:47,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:49,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:51,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:53,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:55,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:41:55,125 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:41:55,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:57,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:41:59,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:01,572 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:42:01,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:04,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:06,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:08,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:10,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:42:10,523 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:42:10,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:13,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:15,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:17,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:19,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:21,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:23,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:25,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:25,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:42:25,795 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:42:27,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:29,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:31,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:31,647 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:42:33,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:35,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:37,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:39,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:41,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:41,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:42:41,017 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:42:43,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:45,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:47,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:49,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:51,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:53,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:55,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:56,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:42:56,263 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:42:57,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:42:59,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:01,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:01,715 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:43:03,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:05,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:07,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:09,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:11,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:11,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:43:11,486 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:43:13,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:15,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:17,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:19,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:21,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:23,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:25,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:26,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:43:26,786 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:43:27,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:29,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:31,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:31,793 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:43:33,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:35,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:37,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:39,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:41,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:42,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:43:42,145 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:43:43,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:45,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:47,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:49,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:51,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:53,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:55,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:57,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:43:57,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:43:57,439 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:43:59,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:01,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:01,867 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:44:03,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:05,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:07,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:09,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:11,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:12,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:44:12,748 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:44:13,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:15,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:17,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:19,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:22,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:24,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:26,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:28,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:44:28,036 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:44:28,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:30,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:31,950 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:44:32,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:34,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:36,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:38,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:40,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:42,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:43,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:44:43,298 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:44:44,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:46,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:48,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:50,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:52,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:54,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:56,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:58,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:44:58,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:44:58,521 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:45:00,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:02,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:45:02,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:04,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:06,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:08,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:10,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:12,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:13,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:45:13,735 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:45:14,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:16,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:18,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:20,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:22,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:24,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:26,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:28,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:29,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:45:29,013 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:45:30,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:32,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:32,102 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:45:34,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:36,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:39,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:41,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:43,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:44,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:45:44,240 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:45:45,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:47,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:49,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:51,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:53,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:55,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:57,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:59,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:45:59,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:45:59,496 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:46:01,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:02,174 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:46:03,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:05,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:07,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:09,100 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:11,100 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:13,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:14,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:46:14,758 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:46:15,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:17,103 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:19,103 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:21,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:23,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:25,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:27,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:29,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:29,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:46:29,982 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:46:31,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:32,248 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:46:33,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:35,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:37,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:39,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:42,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:44,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:45,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:46:45,276 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:46:46,112 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:48,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:50,114 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:52,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:54,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:56,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:46:58,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:00,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:00,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:47:00,515 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:47:02,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:02,320 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:47:04,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:06,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:08,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:10,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:12,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:14,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:15,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:47:15,739 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:47:16,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:18,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:20,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:22,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:24,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:26,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:28,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:30,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:31,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:47:31,007 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:47:32,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:32,393 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:47:34,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:36,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:38,132 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:40,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:42,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:44,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:46,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:46,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:47:46,376 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:47:48,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:50,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:52,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:54,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:56,140 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:47:59,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:01,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:01,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:48:01,598 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:48:02,465 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:48:03,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:05,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:07,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:09,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:11,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:13,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:15,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:16,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:48:16,843 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:48:17,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:19,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:21,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:23,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:25,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:27,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:29,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:31,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:32,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:48:32,148 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:48:32,544 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:48:33,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:35,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:37,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:39,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:41,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:43,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:45,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:47,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:47,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:48:47,507 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:48:49,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:51,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:53,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:55,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:57,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:48:59,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:01,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:02,619 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:49:02,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:49:02,730 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:49:03,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:05,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:07,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:09,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:11,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:13,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:15,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:17,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:17,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:49:17,967 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:49:19,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:21,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:23,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:25,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:28,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:30,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:32,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:32,689 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:49:33,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:49:33,353 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:49:34,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:36,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:38,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:40,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:42,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:44,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:46,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:48,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:48,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:49:48,574 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:49:50,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:52,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:54,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:56,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:49:58,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:00,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:02,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:02,765 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:50:03,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:50:03,796 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:50:04,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:06,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:09,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:11,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:13,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:15,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:17,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:19,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:50:19,098 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:50:19,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:21,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:23,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:25,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:27,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:29,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:31,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:32,840 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:50:33,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:34,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:50:34,475 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:50:35,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:37,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:39,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:41,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:43,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:45,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:47,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:49,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:49,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:50:49,714 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:50:51,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:53,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:55,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:57,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:50:59,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:01,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:02,914 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:51:03,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:05,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:51:05,048 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:51:05,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:07,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:10,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:12,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:14,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:16,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:18,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:20,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:20,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:51:20,342 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:51:22,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:24,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:26,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:28,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:30,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:32,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:32,989 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:51:34,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:35,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:51:35,626 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:51:36,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:38,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:40,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:42,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:44,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:46,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:48,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:50,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:50,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:51:50,862 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:51:52,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:54,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:56,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:51:58,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:00,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:02,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:03,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:52:04,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:06,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:52:06,164 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:52:06,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:08,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:10,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:12,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:14,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:16,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:18,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:20,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:21,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:52:21,419 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:52:22,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:24,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:26,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:28,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:30,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:32,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:33,138 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:52:34,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:36,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:52:36,641 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:52:37,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:39,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:41,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:43,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:45,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:47,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:49,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:51,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:51,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:52:51,865 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:52:53,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:55,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:57,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:52:59,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:01,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:03,214 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:53:03,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:05,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:07,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:53:07,085 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:53:07,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:09,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:11,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:13,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:15,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:17,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:19,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:21,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:22,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:53:22,372 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:53:23,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:25,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:27,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:29,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:31,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:33,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:33,291 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:53:35,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:37,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:37,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:53:37,594 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:53:39,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:41,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:43,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:45,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:47,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:49,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:51,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:52,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:53:52,899 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:53:53,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:55,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:57,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:53:59,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:01,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:03,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:03,367 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:54:05,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:07,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:08,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:54:08,149 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:54:09,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:11,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:13,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:15,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:17,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:19,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:21,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:23,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:23,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:54:23,373 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:54:25,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:27,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:30,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:32,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:33,439 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:54:34,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:36,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:38,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:38,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:54:38,694 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:54:40,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:42,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:44,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:46,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:48,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:50,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:52,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:53,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:54:53,996 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:54:54,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:56,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:54:58,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:00,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:02,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:03,513 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:55:04,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:06,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:08,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:09,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:55:09,338 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:55:10,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:12,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:14,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:16,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:18,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:20,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:22,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:24,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:24,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:55:24,560 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:55:26,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:28,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:30,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:32,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:33,581 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:55:34,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:36,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:38,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:39,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:55:39,933 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:55:40,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:42,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:44,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:46,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:49,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:51,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:53,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:55,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:55:55,155 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:55:55,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:57,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:55:59,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:01,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:03,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:03,649 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:56:05,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:07,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:09,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:10,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:56:10,400 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:56:11,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:13,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:15,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:17,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:19,355 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:21,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:23,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:25,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:25,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:56:25,716 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:56:27,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:29,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:31,361 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:33,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:33,718 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:56:35,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:37,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:39,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:40,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:56:40,940 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:56:41,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:43,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:45,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:47,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:49,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:51,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:53,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:55,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:56,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:56:56,184 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:56:57,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:56:59,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:01,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:03,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:03,789 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:57:05,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:07,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:09,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:11,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:11,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:57:11,611 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:57:13,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:15,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:17,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:19,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:21,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:23,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:25,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:26,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:57:26,853 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:57:27,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:29,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:31,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:33,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:33,860 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:57:35,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:37,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:39,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:41,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:42,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:57:42,066 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:57:43,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:45,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:47,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:49,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:51,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:53,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:55,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:57,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:57:57,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:57:57,449 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:57:59,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:01,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:03,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:03,925 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:58:05,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:07,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:09,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:11,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:12,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:58:12,672 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:58:13,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:15,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:18,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:20,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:22,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:24,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:26,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:27,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:58:27,974 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:58:28,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:30,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:32,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:34,000 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:58:34,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:36,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:38,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:40,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:42,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:43,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:58:43,227 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:58:44,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:46,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:48,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:50,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:52,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:54,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:56,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:58,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:58:58,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:58:58,556 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:59:00,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:02,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:04,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:59:04,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:06,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:08,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:10,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:12,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:13,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:59:13,892 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:59:14,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:17,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:19,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:21,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:23,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:25,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:27,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:29,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:59:29,138 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:59:29,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:31,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:33,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:34,151 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 17:59:35,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:37,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:39,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:41,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:43,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:44,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:59:44,382 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 17:59:45,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:47,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:49,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:51,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:53,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:55,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:57,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:59,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 17:59:59,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 17:59:59,688 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:00:01,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:03,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:04,224 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:00:05,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:07,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:09,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:11,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:13,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:14,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:00:14,985 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:00:15,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:17,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:19,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:21,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:23,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:25,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:27,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:29,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:30,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:00:30,260 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:00:31,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:33,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:34,298 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:00:35,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:37,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:39,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:41,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:43,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:45,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:00:45,562 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:00:46,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:48,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:50,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:52,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:54,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:56,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:00:58,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:00,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:00,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:01:00,789 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:01:02,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:04,369 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:01:04,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:06,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:08,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:10,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:12,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:14,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:16,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:01:16,065 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:01:16,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:18,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:20,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:22,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:24,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:26,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:28,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:30,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:31,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:01:31,282 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:01:32,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:34,443 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:01:34,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:36,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:38,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:40,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:42,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:44,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:46,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:46,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:01:46,635 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:01:48,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:50,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:52,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:54,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:57,506 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:01:59,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:01,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:01,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:02:01,858 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:02:03,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:04,522 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:02:05,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:07,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:09,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:11,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:13,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:15,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:17,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:17,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:02:17,730 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:02:19,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:21,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:23,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:25,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:27,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:29,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:31,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:33,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:02:33,044 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:02:33,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:34,601 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:02:35,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:37,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:39,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:41,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:43,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:45,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:47,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:48,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:02:48,366 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:02:49,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:51,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:53,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:55,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:57,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:02:59,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:01,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:03,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:03,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:03:03,590 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:03:04,676 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:03:05,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:07,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:09,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:11,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:13,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:16,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:18,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:18,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:03:18,958 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:03:20,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:22,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:24,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:26,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:28,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:30,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:32,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:34,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:03:34,257 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:03:34,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:34,753 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:03:36,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:38,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:40,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:42,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:44,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:46,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:48,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:49,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:03:49,492 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:03:50,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:52,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:54,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:56,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:03:58,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:00,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:02,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:04,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:04,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:04:04,787 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:04:05,012 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:04:06,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:08,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:10,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:12,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:14,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:16,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:18,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:20,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:04:20,117 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:04:20,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:22,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:24,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:26,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:28,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:30,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:32,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:34,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:34,902 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:04:35,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:04:35,342 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:04:36,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:38,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:40,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:42,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:44,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:46,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:48,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:50,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:50,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:04:50,638 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:04:53,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:55,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:57,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:04:59,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:01,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:03,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:04,976 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:05:05,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:05,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:05:05,863 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:05:07,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:09,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:11,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:13,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:15,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:17,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:19,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:21,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:05:21,152 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:05:21,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:23,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:25,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:27,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:29,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:31,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:33,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:35,051 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:05:35,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:36,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:05:36,391 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:05:37,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:39,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:41,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:43,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:45,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:47,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:49,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:51,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:05:51,611 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:05:51,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:53,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:55,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:57,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:05:59,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:01,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:03,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:05,129 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:06:05,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:06,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:06:06,900 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:06:07,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:09,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:11,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:13,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:15,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:17,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:19,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:21,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:22,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:06:22,176 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:06:23,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:25,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:28,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:30,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:32,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:34,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:35,207 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:06:36,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:37,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:06:37,484 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:06:38,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:40,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:42,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:44,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:46,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:48,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:50,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:52,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:52,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:06:52,821 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:06:54,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:56,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:06:58,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:00,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:02,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:04,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:05,272 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:07:06,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:08,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:07:08,062 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:07:08,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:10,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:12,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:14,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:16,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:18,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:20,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:22,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:23,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:07:23,352 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:07:24,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:26,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:28,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:30,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:32,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:34,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:35,342 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:07:36,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:38,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:07:38,576 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:07:38,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:40,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:42,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:44,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:46,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:48,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:50,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:52,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:53,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:07:53,888 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:07:54,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:56,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:07:58,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:00,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:02,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:04,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:05,413 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:08:06,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:08,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:09,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:08:09,199 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:08:10,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:12,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:14,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:16,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:18,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:20,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:23,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:24,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:08:24,443 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:08:25,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:27,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:29,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:31,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:33,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:35,483 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:08:35,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:37,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:39,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:08:39,684 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:08:39,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:41,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:43,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:45,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:47,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:49,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:51,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:53,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:54,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:08:54,982 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:08:55,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:57,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:08:59,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:01,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:03,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:05,557 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:09:05,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:07,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:09,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:10,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:09:10,263 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:09:11,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:13,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:15,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:17,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:19,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:21,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:23,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:25,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:09:25,504 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:09:25,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:27,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:29,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:31,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:33,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:35,621 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:09:36,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:38,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:40,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:40,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:09:40,719 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:09:42,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:44,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:46,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:48,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:50,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:52,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:54,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:55,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:09:55,974 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:09:56,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:09:58,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:00,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:02,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:04,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:05,693 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:10:06,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:08,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:10,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:11,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:10:11,185 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:10:12,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:14,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:16,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:18,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:20,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:22,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:24,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:26,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:10:26,455 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:10:26,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:28,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:30,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:32,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:34,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:35,762 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:10:36,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:38,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:40,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:41,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:10:41,751 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:10:42,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:44,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:46,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:48,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:50,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:52,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:55,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:56,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:10:56,965 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:10:57,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:10:59,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:01,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:03,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:05,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:05,829 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:11:07,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:09,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:11,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:12,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:11:12,308 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:11:13,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:15,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:17,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:19,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:21,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:23,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:25,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:27,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:11:27,670 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:11:27,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:29,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:31,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:34,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:35,898 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:11:36,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:38,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:40,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:42,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:42,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:11:42,890 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:11:44,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:46,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:48,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:50,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:52,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:54,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:56,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:11:58,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:11:58,178 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:11:58,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:00,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:02,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:04,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:05,969 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:12:06,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:08,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:10,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:12,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:13,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:12:13,460 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:12:14,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:16,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:18,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:20,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:22,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:24,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:26,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:28,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:12:28,682 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:12:28,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:30,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:32,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:34,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:36,034 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:12:36,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:39,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:41,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:43,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:44,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:12:44,045 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:12:45,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:47,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:49,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:51,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:53,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:55,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:57,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:12:59,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:12:59,265 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:12:59,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:01,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:03,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:05,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:06,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:13:07,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:09,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:11,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:13,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:14,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:13:14,541 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:13:15,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:17,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:19,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:21,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:23,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:25,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:27,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:29,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:29,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:13:29,922 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:13:31,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:33,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:35,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:36,171 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:13:37,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:39,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:41,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:43,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:45,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:13:45,243 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:13:45,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:47,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:49,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:51,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:53,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:55,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:57,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:13:59,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:00,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:14:00,532 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:14:01,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:03,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:05,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:06,235 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:14:07,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:09,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:11,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:13,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:15,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:14:15,749 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:14:15,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:17,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:19,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:21,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:23,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:25,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:27,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:29,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:31,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:14:31,103 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:14:31,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:33,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:35,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:36,302 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:14:37,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:39,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:41,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:44,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:46,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:14:46,325 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:14:46,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:48,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:50,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:52,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:54,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:56,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:14:58,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:00,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:01,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:15:01,626 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:15:02,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:04,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:06,373 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:15:06,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:08,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:10,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:12,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:14,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:16,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:16,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:15:16,887 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:15:18,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:20,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:22,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:24,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:26,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:28,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:30,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:32,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:15:32,166 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:15:32,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:34,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:36,446 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:15:36,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:38,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:40,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:42,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:44,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:46,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:47,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:15:47,388 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:15:48,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:50,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:52,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:55,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:57,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:15:59,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:01,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:02,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:16:02,637 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:16:03,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:05,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:06,512 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:16:07,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:09,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:11,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:13,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:15,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:17,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:17,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:16:17,918 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:16:19,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:21,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:23,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:25,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:27,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:29,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:31,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:33,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:16:33,195 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:16:33,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:35,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:36,581 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:16:37,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:39,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:41,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:43,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:45,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:47,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:48,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:16:48,424 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:16:49,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:51,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:53,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:55,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:57,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:16:59,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:01,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:03,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:17:03,659 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:17:03,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:05,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:06,655 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:17:08,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:10,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:12,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:14,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:16,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:18,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:19,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:17:19,028 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:17:20,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:22,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:24,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:26,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:28,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:30,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:32,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:34,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:17:34,275 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:17:34,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:36,728 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:17:36,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:38,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:40,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:42,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:44,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:46,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:48,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:49,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:17:49,545 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:17:50,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:52,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:54,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:56,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:17:58,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:00,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:02,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:04,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:18:04,909 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:18:04,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:06,799 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:18:06,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:08,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:10,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:12,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:14,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:16,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:18,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:20,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:18:20,156 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:18:20,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:22,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:24,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:26,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:28,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:30,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:32,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:34,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:35,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:18:35,455 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:18:36,876 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:18:36,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:39,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:41,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:43,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:45,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:47,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:49,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:50,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:18:50,687 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:18:51,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:53,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:55,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:57,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:18:59,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:01,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:03,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:05,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:19:05,934 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:19:05,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:06,950 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:19:07,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:09,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:11,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:13,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:15,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:17,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:19,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:21,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:19:21,207 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:19:21,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:23,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:25,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:27,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:29,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:31,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:33,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:35,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:36,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:19:36,516 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:19:37,025 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:19:37,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:39,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:41,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:43,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:45,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:47,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:49,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:51,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:19:51,742 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:19:51,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:53,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:55,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:57,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:19:59,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:01,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:03,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:06,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:06,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:20:06,996 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:20:07,209 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:20:08,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:10,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:12,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:14,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:16,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:18,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:20,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:22,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:20:22,268 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:20:22,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:24,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:26,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:28,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:30,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:32,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:34,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:36,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:37,171 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:20:37,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:20:37,571 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:20:38,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:40,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:42,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:44,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:46,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:48,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:50,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:52,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:20:52,794 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:20:52,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:54,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:56,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:20:58,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:00,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:02,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:04,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:06,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:07,247 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:21:08,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:21:08,179 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:21:09,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:11,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:13,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:15,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:17,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:19,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:21,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:23,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:23,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:21:23,446 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:21:25,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:27,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:29,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:31,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:34,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:36,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:37,320 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:21:38,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:38,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:21:38,736 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:21:40,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:42,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:44,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:46,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:48,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:50,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:52,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:53,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:21:53,954 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:21:54,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:56,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:21:58,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:00,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:02,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:04,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:06,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:07,394 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:22:08,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:09,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:22:09,233 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:22:10,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:12,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:14,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:16,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:18,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:20,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:22,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:24,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:24,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:22:24,456 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:22:26,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:28,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:30,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:32,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:34,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:36,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:37,473 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:22:38,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:39,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:22:39,770 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:22:40,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:42,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:44,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:46,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:48,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:50,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:52,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:54,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:55,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:22:55,062 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:22:56,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:22:58,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:00,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:02,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:04,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:06,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:07,549 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:23:08,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:10,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:10,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:23:10,324 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:23:12,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:14,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:17,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:19,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:21,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:23,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:25,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:25,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:23:25,589 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:23:27,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:29,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:31,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:33,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:35,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:37,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:37,623 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:23:39,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:40,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:23:40,852 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:23:41,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:43,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:45,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:47,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:49,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:51,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:53,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:55,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:56,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:23:56,075 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:23:57,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:23:59,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:01,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:03,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:05,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:07,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:07,700 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:24:09,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:11,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:11,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:24:11,432 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:24:13,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:15,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:17,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:19,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:21,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:23,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:25,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:26,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:24:26,702 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:24:27,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:29,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:31,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:33,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:36,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:37,777 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:24:38,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:40,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:41,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:24:41,921 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:24:42,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:44,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:46,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:48,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:50,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:52,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:54,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:56,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:24:57,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:24:57,256 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:24:58,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:00,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:02,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:04,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:06,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:07,855 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:25:08,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:10,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:12,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:12,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:25:12,533 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:25:14,100 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:16,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:18,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:20,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:22,103 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:24,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:26,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:27,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:25:27,754 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:25:28,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:30,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:32,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:34,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:36,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:37,932 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:25:38,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:40,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:42,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:44,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:46,112 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:47,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:25:47,754 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:25:48,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:50,114 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:52,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:54,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:56,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:25:58,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:00,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:02,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:03,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:26:03,074 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:26:05,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:07,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:08,006 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:26:09,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:11,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:13,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:15,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:17,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:18,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:26:18,301 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:26:19,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:21,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:23,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:25,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:27,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:29,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:31,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:33,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:33,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:26:33,620 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:26:35,132 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:37,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:38,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:26:39,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:41,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:43,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:45,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:47,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:48,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:26:48,908 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:26:49,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:51,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:53,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:55,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:57,140 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:26:59,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:01,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:03,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:04,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:27:04,132 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:27:05,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:07,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:08,157 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:27:09,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:11,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:13,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:15,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:17,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:19,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:27:19,515 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:27:20,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:22,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:24,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:26,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:28,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:30,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:32,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:34,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:34,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:27:34,890 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:27:36,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:38,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:38,230 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:27:40,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:42,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:44,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:46,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:48,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:50,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:27:50,137 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:27:50,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:52,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:54,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:56,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:27:58,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:00,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:02,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:04,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:05,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:28:05,417 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:28:06,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:08,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:08,305 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:28:10,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:12,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:14,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:16,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:18,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:20,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:20,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:28:20,705 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:28:22,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:24,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:26,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:28,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:31,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:33,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:35,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:35,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:28:35,924 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:28:37,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:38,380 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:28:39,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:41,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:43,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:45,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:47,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:49,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:51,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:51,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:28:51,270 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:28:53,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:55,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:57,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:28:59,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:01,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:03,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:05,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:06,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:29:06,570 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:29:07,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:08,453 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:29:09,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:11,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:13,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:15,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:17,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:19,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:21,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:21,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:29:21,826 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:29:23,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:25,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:27,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:29,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:31,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:33,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:35,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:37,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:29:37,120 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:29:37,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:38,529 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:29:39,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:41,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:43,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:45,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:47,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:49,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:51,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:52,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:29:52,352 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:29:53,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:55,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:29:57,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:00,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:02,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:04,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:06,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:07,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:30:07,574 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:30:08,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:08,604 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:30:10,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:12,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:14,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:16,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:18,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:20,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:22,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:22,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:30:22,791 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:30:24,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:26,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:28,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:30,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:32,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:34,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:36,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:38,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:30:38,015 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:30:38,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:38,679 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:30:40,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:42,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:44,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:46,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:48,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:50,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:52,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:53,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:30:53,319 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:30:54,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:56,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:30:58,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:00,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:02,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:04,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:06,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:08,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:08,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:31:08,533 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:31:08,758 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:31:10,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:12,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:14,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:16,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:18,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:20,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:22,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:23,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:31:23,757 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:31:24,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:26,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:29,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:31,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:33,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:35,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:37,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:38,835 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:31:39,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:31:39,139 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:31:39,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:41,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:43,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:45,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:47,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:49,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:51,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:53,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:54,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:31:54,434 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:31:55,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:57,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:31:59,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:01,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:03,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:05,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:07,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:08,911 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:32:09,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:09,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:32:09,756 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:32:11,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:13,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:15,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:17,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:19,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:21,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:23,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:24,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:32:24,984 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:32:25,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:27,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:29,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:31,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:33,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:35,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:37,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:38,987 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:32:39,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:40,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:32:40,276 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:32:41,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:43,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:45,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:47,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:49,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:51,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:53,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:55,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:55,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:32:55,559 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:32:57,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:32:59,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:01,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:03,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:05,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:07,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:09,063 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:33:09,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:10,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:33:10,782 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:33:11,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:13,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:15,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:17,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:19,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:21,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:24,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:26,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:33:26,091 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:33:26,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:28,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:30,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:32,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:34,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:36,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:38,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:39,136 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:33:40,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:41,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:33:41,379 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:33:42,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:44,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:46,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:48,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:50,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:52,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:54,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:56,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:33:56,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:33:56,658 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:33:58,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:00,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:02,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:04,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:06,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:08,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:09,211 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:34:10,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:11,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:34:11,891 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:34:12,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:14,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:16,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:18,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:20,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:22,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:24,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:26,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:27,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:34:27,121 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:34:28,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:30,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:32,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:34,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:36,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:38,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:39,284 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:34:40,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:42,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:34:42,486 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:34:43,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:45,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:47,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:49,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:51,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:53,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:55,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:57,355 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:34:57,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:34:57,733 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:34:59,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:01,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:03,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:05,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:07,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:09,357 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:35:09,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:11,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:13,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:35:13,050 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:35:13,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:15,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:17,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:19,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:21,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:23,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:25,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:27,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:28,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:35:28,274 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:35:29,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:31,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:33,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:35,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:37,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:39,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:39,434 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:35:41,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:43,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:43,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:35:43,599 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:35:45,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:47,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:49,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:51,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:53,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:55,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:57,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:35:58,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:35:58,889 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:35:59,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:01,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:03,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:05,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:07,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:09,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:09,509 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:36:11,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:13,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:14,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:36:14,154 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:36:15,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:17,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:19,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:21,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:24,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:26,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:28,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:29,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:36:29,378 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:36:30,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:32,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:34,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:36,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:38,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:39,584 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:36:40,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:42,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:44,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:44,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:36:44,638 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:36:46,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:48,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:50,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:52,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:54,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:56,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:58,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:36:59,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:36:59,893 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:37:00,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:02,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:04,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:06,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:08,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:09,654 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:37:10,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:12,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:14,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:15,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:37:15,179 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:37:16,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:18,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:20,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:22,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:24,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:26,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:28,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:30,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:30,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:37:30,467 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:37:32,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:34,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:36,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:38,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:39,729 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:37:40,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:42,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:44,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:45,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:37:45,753 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:37:46,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:48,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:51,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:53,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:55,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:57,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:37:59,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:00,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:38:00,976 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:38:01,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:03,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:05,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:07,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:09,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:09,805 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:38:11,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:13,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:15,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:16,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:38:16,199 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:38:17,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:19,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:21,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:23,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:25,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:27,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:29,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:31,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:38:31,426 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:38:31,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:33,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:35,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:37,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:39,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:39,882 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:38:41,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:43,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:45,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:46,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:38:46,794 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:38:47,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:49,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:51,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:53,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:55,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:57,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:38:59,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:01,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:02,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:39:02,086 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:39:03,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:05,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:07,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:09,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:09,954 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:39:11,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:13,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:15,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:17,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:39:17,342 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:39:17,471 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:19,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:21,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:23,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:26,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:28,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:30,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:32,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:32,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:39:32,569 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:39:34,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:36,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:38,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:40,030 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:39:40,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:42,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:44,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:46,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:47,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:39:47,784 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:39:48,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:50,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:52,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:54,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:56,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:39:58,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:00,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:02,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:03,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:40:03,122 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:40:04,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:06,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:08,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:10,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:40:10,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:12,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:14,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:16,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:18,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:40:18,364 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:40:18,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:20,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:22,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:24,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:26,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:28,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:30,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:33,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:33,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:40:33,683 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:40:35,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:37,506 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:39,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:40,181 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:40:41,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:43,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:45,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:47,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:48,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:40:48,922 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:40:49,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:51,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:53,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:55,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:57,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:40:59,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:01,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:03,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:04,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:41:04,209 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:41:05,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:07,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:09,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:10,257 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:41:11,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:13,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:15,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:17,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:19,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:41:19,439 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:41:19,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:21,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:23,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:25,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:27,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:29,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:31,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:33,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:34,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:41:34,682 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:41:35,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:37,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:39,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:40,330 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:41:41,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:43,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:45,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:48,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:49,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:41:49,897 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:41:50,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:52,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:54,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:56,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:41:58,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:00,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:02,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:04,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:05,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:42:05,203 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:42:06,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:08,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:10,403 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:42:10,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:12,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:14,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:16,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:18,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:20,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:42:20,523 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:42:20,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:22,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:24,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:26,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:28,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:30,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:32,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:34,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:35,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:42:35,747 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:42:36,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:38,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:40,480 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:42:40,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:42,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:44,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:46,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:48,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:50,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:51,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:42:51,104 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:42:52,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:54,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:56,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:42:58,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:00,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:02,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:04,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:06,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:43:06,332 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:43:06,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:09,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:10,553 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:43:11,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:13,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:15,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:17,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:19,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:21,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:21,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:43:21,679 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:43:23,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:25,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:27,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:29,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:31,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:33,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:35,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:36,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:43:36,959 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:43:37,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:39,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:40,632 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:43:41,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:43,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:45,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:47,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:49,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:51,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:52,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:43:52,393 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:43:53,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:55,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:57,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:43:59,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:01,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:03,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:05,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:07,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:07,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:44:07,711 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:44:09,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:10,707 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:44:11,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:13,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:15,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:17,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:19,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:21,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:22,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:44:22,980 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:44:23,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:25,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:27,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:29,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:31,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:33,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:35,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:37,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:38,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:44:38,195 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:44:39,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:40,782 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:44:41,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:43,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:46,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:48,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:50,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:52,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:53,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:44:53,544 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:44:54,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:56,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:44:58,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:00,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:02,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:04,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:06,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:08,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:08,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:45:08,803 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:45:10,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:10,856 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:45:12,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:14,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:16,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:18,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:20,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:22,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:24,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:45:24,095 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:45:24,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:26,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:28,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:30,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:32,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:34,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:36,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:38,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:39,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:45:39,323 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:45:40,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:40,932 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:45:42,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:44,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:46,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:48,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:50,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:52,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:54,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:54,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:45:54,645 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:45:56,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:45:58,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:00,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:02,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:04,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:07,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:09,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:09,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:46:09,914 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:46:11,002 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:46:11,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:13,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:15,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:17,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:19,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:21,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:23,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:25,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:46:25,134 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:46:25,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:27,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:29,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:31,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:33,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:35,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:37,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:39,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:40,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:46:40,448 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:46:41,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:46:41,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:43,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:45,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:47,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:49,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:51,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:53,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:55,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:55,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:46:55,748 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:46:57,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:46:59,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:01,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:03,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:06,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:08,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:10,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:10,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:47:10,969 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:47:11,228 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:47:12,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:14,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:16,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:18,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:20,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:22,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:24,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:26,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:47:26,317 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:47:26,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:28,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:30,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:32,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:34,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:36,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:38,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:40,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:41,218 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:47:41,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:47:41,541 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:47:42,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:44,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:46,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:48,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:50,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:52,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:54,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:56,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:47:56,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:47:56,838 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:47:58,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:00,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:02,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:04,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:06,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:08,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:10,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:11,294 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:48:12,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:48:12,186 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:48:12,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:14,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:16,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:18,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:20,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:22,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:24,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:27,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:48:27,408 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:48:27,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:29,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:31,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:33,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:35,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:37,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:39,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:41,368 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:48:41,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:42,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:48:42,635 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:48:43,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:45,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:47,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:49,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:51,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:53,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:55,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:57,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:48:58,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:48:58,007 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:48:59,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:01,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:03,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:05,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:07,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:09,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:11,441 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:49:11,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:13,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:49:13,342 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:49:13,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:15,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:17,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:19,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:21,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:23,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:25,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:27,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:28,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:49:28,598 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:49:29,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:31,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:33,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:35,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:37,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:39,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:41,518 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:49:41,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:43,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:43,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:49:43,819 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:49:45,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:47,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:49,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:52,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:54,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:56,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:58,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:49:59,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:49:59,039 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:50:00,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:50:11,595 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:50:14,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:50:14,261 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:50:29,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:50:29,521 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:50:41,673 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:50:44,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:50:44,791 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:51:00,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:51:00,069 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:51:11,748 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:51:15,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:51:15,327 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:51:17,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:51:17,089 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 18:51:17,143 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 18:51:17,221 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 18:51:17,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 18:51:18,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:51:20,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:51:24,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:51:30,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:51:30,604 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:51:40,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:51:41,830 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:51:45,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:51:45,832 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:52:01,057 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:52:01,057 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:52:11,906 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:52:16,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:52:16,281 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:52:31,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:52:31,507 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:52:41,984 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:52:46,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:52:46,730 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:53:01,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:53:01,952 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:53:12,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:53:17,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:53:17,179 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:53:32,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:53:32,403 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:53:42,138 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:53:47,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:53:47,636 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:54:02,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:54:02,858 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:54:12,214 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:54:18,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:54:18,081 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:54:33,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:54:33,428 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:54:42,291 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:54:48,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:54:48,651 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:55:03,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:55:03,891 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:55:12,370 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:55:19,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:55:19,115 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:55:30,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:55:32,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:55:34,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:55:34,337 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:55:34,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:34,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:38,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:55:42,446 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:55:42,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:42,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:55:46,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:46,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:55:48,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:48,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:49,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:55:49,574 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:55:50,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:50,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:55:52,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:52,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:54,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:55:55,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,187 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:55,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:56,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:55:58,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,534 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,536 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:55:58,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:00,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:01,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:01,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:02,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:03,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:04,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:56:04,791 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:56:04,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:06,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:07,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:07,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:09,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:10,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:12,518 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:56:12,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:14,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:16,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:16,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:18,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:20,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:56:20,013 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:56:20,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:21,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:21,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:22,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:26,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:28,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:31,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:33,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:34,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:34,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:35,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:56:35,360 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:56:35,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:37,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:40,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:40,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:56:41,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:42,588 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:56:43,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:45,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:47,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:49,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:50,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:56:50,583 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:56:51,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:53,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:55,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:57,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:56:59,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:01,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:03,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:03,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:05,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:57:05,812 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:57:05,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:07,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:07,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:08,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:08,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:09,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:10,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:10,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:11,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:12,661 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:57:13,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:13,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:15,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:15,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:16,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:16,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:17,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:18,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:18,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:18,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:18,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:19,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:21,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:57:21,033 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:57:21,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:21,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:23,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:23,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:25,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:26,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:26,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:29,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:30,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:30,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:33,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:34,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:34,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:36,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:57:36,258 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:57:37,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:37,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:40,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:40,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:41,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:42,737 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:57:42,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:42,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:43,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:44,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,410 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:44,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:47,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:48,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:48,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:50,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:57:51,660 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:57:51,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:51,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:53,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,632 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,635 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:53,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:55,006 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:55,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:56,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:57,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:57:59,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:57:59,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:00,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:00,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:01,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:03,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:03,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:05,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:06,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:58:06,904 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:58:07,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:08,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:08,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:09,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:12,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:12,814 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:58:13,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:15,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:17,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:17,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:19,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:21,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:22,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:58:22,190 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:58:23,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:25,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:26,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:26,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:27,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:29,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:30,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:30,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:31,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:33,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:35,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:37,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:58:37,411 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:58:37,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:39,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:41,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:42,890 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:58:43,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:45,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:47,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:49,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:51,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:52,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:58:52,658 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:58:53,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,821 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:53,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:55,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:55,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:56,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:57,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,278 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:57,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,633 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,634 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,637 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,638 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,639 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,643 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:58,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:58:59,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:58:59,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:00,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:02,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:02,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:04,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:04,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:05,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:05,469 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 18:59:05,684 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 18:59:05,759 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 18:59:05,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 18:59:06,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:07,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:08,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:59:08,160 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:59:12,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:12,988 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:59:13,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:16,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:16,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:17,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:18,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:18,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:21,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:21,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:22,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:59:23,566 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:59:23,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:23,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:25,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:27,078 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,083 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,084 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:27,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:29,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,357 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:29,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:31,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:31,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:32,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:35,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:35,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:38,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:59:38,791 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:59:39,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:40,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:40,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:41,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:41,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:42,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:43,065 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 18:59:44,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,416 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:44,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:45,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,870 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:45,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:47,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:49,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:51,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:51,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:52,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:53,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:54,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 18:59:54,010 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 18:59:54,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:54,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:55,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,642 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:55,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:57,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 18:59:59,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 18:59:59,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:01,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:01,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:03,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:03,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:05,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:06,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:06,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:07,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:09,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:09,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:00:09,240 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:00:13,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:13,139 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:00:15,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:15,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:15,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:17,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:19,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:21,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:22,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:22,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:23,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:24,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:00:24,540 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:00:25,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:27,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:29,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:31,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:33,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:35,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:37,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:39,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:39,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:00:39,763 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:00:41,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:43,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:43,209 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:00:45,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:46,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:46,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:47,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:47,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:48,930 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:49,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:50,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:50,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:51,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:52,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:52,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:00:53,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:53,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:00:55,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:00:55,036 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:00:55,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:01,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:03,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:03,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:05,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:06,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,921 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:06,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:09,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:10,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:01:10,261 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:01:10,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:10,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:12,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:13,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:13,289 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:01:14,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:14,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:17,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:18,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:18,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:19,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:21,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:22,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:23,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,471 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,472 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:23,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:24,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:25,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:01:25,552 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:01:26,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:26,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:28,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:30,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:30,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:30,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,462 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:32,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,545 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:33,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:34,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:35,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:35,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:36,543 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:36,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:38,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:39,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:39,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:40,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:01:40,777 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:01:41,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:41,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:42,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:42,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:43,362 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:01:44,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:46,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:46,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,801 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:46,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:48,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:49,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:50,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:52,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:54,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:55,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:55,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:01:56,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:01:56,082 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:01:56,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:56,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:57,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:01:58,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:00,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:00,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:00,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:02,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:05,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,806 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:05,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:06,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:08,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:10,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:11,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:02:11,368 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:02:12,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:12,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:12,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:13,431 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:02:14,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:16,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:18,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:20,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:22,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:24,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:26,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:26,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:02:26,616 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:02:28,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:30,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:32,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:34,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:36,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:36,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:36,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:37,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:38,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,497 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:38,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:39,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:40,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:41,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:41,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:02:41,844 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:02:41,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:42,119 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 19:02:42,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:42,191 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 19:02:42,249 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 19:02:43,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 19:02:43,553 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:02:44,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:46,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,307 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,309 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:46,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:48,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:54,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:56,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:02:57,315 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:02:57,315 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:02:58,541 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,546 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,547 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,549 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,550 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,551 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:02:58,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:01,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:04,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:04,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:05,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:07,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,815 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,816 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,819 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:07,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:08,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:09,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:10,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,529 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:10,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:11,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:12,544 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:03:12,544 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:03:13,628 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:03:14,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:14,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:15,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:15,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,465 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:16,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:17,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:17,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,919 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,920 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:19,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:21,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,320 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:21,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,424 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:22,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:23,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:24,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:24,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:25,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:27,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:27,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:03:27,842 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:03:29,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:29,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:29,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:31,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:31,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:33,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:35,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:35,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:37,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,891 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:37,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:38,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:39,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:40,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,353 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:40,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:41,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:43,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:03:43,116 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:03:43,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:43,697 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:03:45,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:47,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:49,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:49,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:50,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:51,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,426 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,427 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,428 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,431 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:51,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:53,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:54,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:54,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:55,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:57,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:58,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:03:58,341 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:03:59,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:03:59,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:03:59,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:01,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:02,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:02,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:03,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:05,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:07,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:09,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:11,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:13,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:13,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:04:13,676 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:04:13,919 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:04:15,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:17,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:19,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:21,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:23,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:25,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:26,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,292 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,293 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:26,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:27,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:28,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:28,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:04:28,948 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:04:29,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:29,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:29,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:29,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:29,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:29,548 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:29,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:29,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:29,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:29,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:29,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:29,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:29,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:31,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:32,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,091 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:32,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:36,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:37,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:38,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:38,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:39,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:39,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:39,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:39,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:39,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:39,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:39,997 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:39,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:39,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,007 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:40,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:43,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:43,848 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:04:44,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:04:44,170 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:04:48,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:52,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:52,280 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,286 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,287 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,288 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:52,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:56,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:58,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:04:58,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,825 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:58,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:04:59,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:04:59,405 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:05:02,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:02,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:03,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:03,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:04,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:04,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,614 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:06,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:07,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:08,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:09,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,487 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:09,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:10,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:11,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:11,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,958 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:12,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:13,925 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:05:14,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:14,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,681 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,690 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:14,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:05:14,706 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:05:15,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,612 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,613 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,947 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:15,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:16,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:17,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:17,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:18,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:20,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:20,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,689 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,693 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:20,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:23,173 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:24,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,247 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:24,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:26,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,765 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:26,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:27,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:28,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:28,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:29,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:30,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:05:30,033 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:05:30,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:31,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,100 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,163 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,164 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,167 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,168 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:31,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:32,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:32,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,463 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,464 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,468 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,470 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,473 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,475 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:33,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:34,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:36,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:38,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:39,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:39,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:40,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:42,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:42,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,316 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,317 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,318 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,319 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,323 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,324 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:42,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:43,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:44,000 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:05:44,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:45,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:45,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:05:45,268 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:05:46,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:48,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:50,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:51,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:51,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:52,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:53,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:53,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:05:54,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:56,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:05:58,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:00,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:00,641 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:06:00,641 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:06:02,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:04,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:06,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:08,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:10,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:12,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:12,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,388 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:12,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:13,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:06:14,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,190 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:14,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,454 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,555 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,560 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,561 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:14,816 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 19:06:15,053 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 19:06:15,123 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 19:06:15,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 19:06:15,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:06:15,861 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:06:16,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:17,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:22,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:26,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:26,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:29,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:29,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:30,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:31,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:06:31,203 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:06:31,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,310 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,313 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,314 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,939 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,940 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,948 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:31,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:34,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:38,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:40,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:44,198 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:06:44,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:46,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:46,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:06:46,433 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:06:49,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,402 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,403 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,404 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,405 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,411 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:49,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:50,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:50,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:52,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:56,165 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,166 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,169 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,171 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,172 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,178 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,179 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:56,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:57,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:57,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,048 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:58,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:06:59,321 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,322 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,325 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,326 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,327 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:06:59,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:00,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,582 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:00,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:01,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:07:01,655 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:07:02,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:02,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:02,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:02,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:02,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:02,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:02,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:02,996 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,001 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,170 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,174 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,175 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,185 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,186 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,328 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,329 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,334 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:03,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:04,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,588 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:04,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,556 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,557 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:05,611 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:06,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:06,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:08,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:09,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,449 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,455 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:09,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:10,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:10,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:12,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,837 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:12,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:14,275 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:07:15,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:15,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:16,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,391 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,393 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,400 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,401 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,408 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,409 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,412 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,413 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,414 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,415 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,418 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,419 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,420 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,421 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,422 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,520 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:16,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:07:16,934 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:07:18,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:18,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,371 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,652 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:18,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:20,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:21,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:21,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:22,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:23,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:23,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:24,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:25,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:27,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:27,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:27,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:29,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:31,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:32,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:07:32,176 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:07:32,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,456 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,457 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,458 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,459 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,460 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,494 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:32,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:33,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:35,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:36,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:36,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:37,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:39,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:41,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:42,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:42,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:43,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:44,348 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:07:45,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,260 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:45,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:47,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:47,474 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:07:47,474 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:07:49,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:51,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:53,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:55,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:57,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:59,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:07:59,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:07:59,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:00,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:00,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:00,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:00,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:01,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:01,012 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:01,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:01,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:01,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:01,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:01,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:01,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:01,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:02,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:08:02,709 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:08:03,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:03,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,530 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:03,532 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,177 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,180 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:05,197 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:09,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,578 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,579 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,580 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,583 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:09,584 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:13,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:14,420 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:08:15,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:18,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:08:18,072 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:08:19,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:23,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:25,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,514 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,516 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,517 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,518 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,522 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,526 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,527 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:25,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,107 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:26,108 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:27,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,385 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:27,387 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:29,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:33,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:33,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:08:33,376 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:08:35,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:38,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,938 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,941 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,942 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,943 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,944 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,945 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,949 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,952 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,953 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:38,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:39,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:41,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:43,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:44,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,192 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,200 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:44,496 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:08:46,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,766 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:46,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:47,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:48,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,496 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,502 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,503 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:48,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:08:48,702 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:08:49,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:49,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,430 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,435 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,436 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,443 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:49,444 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:50,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:50,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:50,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:50,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:50,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:50,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:50,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:50,995 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:50,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:50,998 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:51,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:51,000 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:51,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:51,002 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:51,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:51,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:51,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:52,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,056 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,094 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:52,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,236 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,237 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,239 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:53,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:54,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,355 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,361 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,362 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,369 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:54,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:56,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:57,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:59,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,045 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,046 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,052 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,053 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,054 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:08:59,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,335 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:08:59,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:01,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:02,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:02,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:03,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:03,946 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:09:03,946 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:09:04,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,886 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,887 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,888 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:04,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,009 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,010 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:05,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:07,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,161 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,162 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:07,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,959 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:07,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:08,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:09,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,376 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:09,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:10,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,993 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:11,994 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:12,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,135 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,139 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:13,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:14,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,223 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:14,569 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:09:15,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:17,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,093 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,096 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,097 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,099 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,101 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,102 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,103 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,104 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,105 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:17,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:19,311 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:09:19,312 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:09:19,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:21,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:22,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,662 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:22,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:23,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:25,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:26,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:26,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:27,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:29,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,070 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:29,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:31,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:33,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:33,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,661 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,669 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,670 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,672 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,673 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:33,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:34,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:09:34,567 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:09:35,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:37,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:39,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:41,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:43,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:44,642 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:09:45,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:46,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:46,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,478 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,640 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,644 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,646 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,647 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:47,899 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 19:09:48,076 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 19:09:48,147 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 19:09:48,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:48,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 19:09:49,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:09:49,790 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:09:50,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:54,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:09:54,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:54,773 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,210 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,216 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,217 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:09:56,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:00,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:02,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,254 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:02,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:05,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:10:05,144 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:10:06,350 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:10,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:14,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:14,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,581 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:14,729 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:10:16,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:17,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,382 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,384 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:17,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,807 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,818 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,820 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,822 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:18,824 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:20,383 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:10:20,383 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:10:20,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,513 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,515 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,519 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,521 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,523 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:20,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:22,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:24,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,862 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,873 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,878 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:24,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:26,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:28,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:30,360 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:33,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:33,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:34,362 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:35,433 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,434 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,438 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,439 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,442 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,446 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,448 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,450 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,451 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:10:35,605 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:10:35,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,858 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,859 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,863 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,865 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,867 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,868 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:35,869 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:36,363 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:37,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,128 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:37,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,276 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,279 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,281 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,283 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,284 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:38,364 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:39,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,340 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,341 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,342 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,351 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:39,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,587 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,594 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:40,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:42,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:44,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:44,806 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:10:45,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,567 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,576 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,577 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:45,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:46,476 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,481 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:46,491 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,772 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,912 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,913 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,914 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,915 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:47,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,138 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,144 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:48,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,851 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:48,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,028 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,037 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,191 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,194 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,196 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,198 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:49,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:50,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:50,828 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:10:50,829 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:10:52,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:53,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,291 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,295 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,296 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,301 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,304 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:53,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:54,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,586 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,593 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,645 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,650 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,651 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:54,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:56,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:57,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,020 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:58,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:10:59,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,129 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,133 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:10:59,134 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,062 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,069 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,072 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,073 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:00,889 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,890 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,899 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,900 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:00,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,224 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,226 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,231 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,232 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,233 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,234 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:01,235 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,137 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,140 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:02,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,986 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:02,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,836 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,842 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:03,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,032 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:04,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,893 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,895 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,896 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,898 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,901 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,902 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,903 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,904 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,905 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,906 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,966 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,971 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,976 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:04,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:06,050 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:11:06,050 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:11:06,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:07,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,692 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,694 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,695 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,696 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,697 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,779 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,785 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,792 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:07,793 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:08,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:10,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:12,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:14,030 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,033 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,038 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,040 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,041 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,042 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,043 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:14,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:14,884 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:11:15,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:15,631 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,298 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,299 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,300 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,303 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,305 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,306 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:16,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:18,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:19,240 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,241 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,248 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,249 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,251 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:19,252 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:20,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:21,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:11:21,290 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:11:22,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:24,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:25,923 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,924 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,926 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,928 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,929 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,931 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,933 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,934 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:25,936 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:26,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:28,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:30,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:32,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:34,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:35,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:35,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:35,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:35,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:35,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:35,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:35,243 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:35,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:35,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:35,245 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:35,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:35,246 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:36,388 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:36,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:11:36,506 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:11:40,358 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,359 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,367 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,374 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,375 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,377 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,379 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,380 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:40,381 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:42,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:44,960 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:11:46,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:49,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,767 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,768 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,769 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,771 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,775 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,776 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:49,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,272 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,275 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:50,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:51,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:11:51,727 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:11:54,396 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:56,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:11:56,826 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,830 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,831 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,832 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,833 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,834 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,841 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:11:56,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:00,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:04,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:06,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:07,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:12:07,024 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:12:09,558 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,562 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,563 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,569 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,570 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,571 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,573 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,574 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,575 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:09,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:11,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:13,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:13,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,708 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:13,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:15,037 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:12:15,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:19,332 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,333 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,337 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,339 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,343 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,344 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,345 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,349 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:19,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:21,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:22,255 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:12:22,256 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:12:23,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,653 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,736 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,737 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:23,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:25,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:27,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:28,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,592 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,595 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,596 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,597 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,604 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:28,610 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:29,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:30,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,218 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,219 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,221 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,225 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,228 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,229 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,265 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,269 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,273 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:30,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:33,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,615 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,626 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:33,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:34,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:35,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,802 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,804 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,805 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,810 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,811 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,812 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:35,813 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,110 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,111 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,115 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,121 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:37,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:37,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:12:37,483 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:12:39,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:39,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,495 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,498 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,499 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,500 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,501 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,504 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,506 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,508 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:39,510 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,789 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,794 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,795 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,796 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,797 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,799 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:40,800 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:41,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,479 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,484 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,485 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,488 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,490 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,492 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:41,493 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,781 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,783 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,784 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,786 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,787 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,790 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,791 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,843 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,844 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,846 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,847 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,848 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,850 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,853 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,854 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,855 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:42,857 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:43,616 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,619 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,620 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,621 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,622 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,625 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,628 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,629 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,630 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,655 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,657 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,659 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,660 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,663 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,664 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,665 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,666 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,667 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,668 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:43,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,011 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,015 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,021 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,950 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,951 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,954 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,955 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,957 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,960 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,961 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,964 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:44,965 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:45,115 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:12:45,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:46,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,969 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,972 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,973 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,975 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,979 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:46,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,055 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,058 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,059 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,060 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,061 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,064 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,065 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,066 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,068 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,142 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,145 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,148 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,154 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:47,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:49,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:50,075 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,076 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,079 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,082 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,085 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,087 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,088 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,089 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,147 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,151 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,153 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,155 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,156 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,157 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,158 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,159 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:50,160 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,199 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,202 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,203 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,207 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,208 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,209 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,212 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,257 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,262 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,264 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,267 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,268 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,270 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,271 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:51,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,871 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,874 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,876 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,879 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,881 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,882 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,883 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:51,884 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:52,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:12:52,744 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:12:53,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:53,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,740 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,742 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,752 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,978 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,981 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,982 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,984 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,987 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,988 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,989 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,990 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,991 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:53,992 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,013 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,016 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,018 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,022 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,024 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,025 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:54,027 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:55,700 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,701 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,703 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,704 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,705 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,706 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,709 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,713 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,759 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,761 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,762 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:55,763 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,712 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,718 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:56,725 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:57,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:12:59,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,386 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,389 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,390 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,392 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,394 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,395 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,396 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,397 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,398 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,399 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:12:59,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:01,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:02,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,114 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,117 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,118 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,122 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,123 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,124 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,126 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:02,127 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:03,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,743 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,746 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,751 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,753 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,754 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,755 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:03,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:05,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:06,674 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,675 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,677 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,678 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,679 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,680 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,682 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,684 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,685 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,686 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,714 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,716 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,717 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,719 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,720 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,721 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:06,724 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:07,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:07,968 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:13:07,969 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:13:09,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:11,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:12,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,601 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,602 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,603 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,605 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,606 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,607 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:12,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:13,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:14,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,723 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,726 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,727 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,728 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,730 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,731 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,732 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,733 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,734 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:14,735 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:15,194 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:13:15,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:17,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:19,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:21,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:23,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,035 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,176 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,181 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,182 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,183 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,184 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:13:23,190 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:13:23,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 19:13:23,474 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 19:13:23,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:23,667 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 19:13:23,741 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 19:13:24,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:24,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 19:13:25,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:27,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:29,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:31,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:33,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:35,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:37,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:38,538 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:13:38,540 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:13:39,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:41,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:43,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:45,317 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:13:45,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:47,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:49,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:51,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:53,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:53,756 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:13:53,756 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:13:55,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:57,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:13:59,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:01,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:03,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:05,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:07,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:08,977 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:14:08,977 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:14:09,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:11,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:14,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:15,377 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:14:16,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:18,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:20,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:22,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:24,214 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:14:24,214 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:14:24,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:26,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:28,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:30,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:32,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:34,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:36,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:38,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:39,440 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:14:39,441 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:14:40,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:42,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:44,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:45,448 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:14:46,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:48,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:50,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:52,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:54,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:54,688 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:14:54,688 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:14:56,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:14:58,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:00,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:02,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:04,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:06,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:08,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:09,907 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:15:09,907 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:15:10,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:12,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:14,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:15,522 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:15:16,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:18,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:20,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:22,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:24,588 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:25,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:15:25,222 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:15:26,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:28,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:30,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:32,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:34,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:36,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:38,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:40,537 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:15:40,538 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:15:41,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:43,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:45,590 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:15:45,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:47,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:49,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:51,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:53,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:55,601 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:55,780 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:15:55,780 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:15:57,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:15:59,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:01,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:03,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:05,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:07,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:09,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:11,003 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:16:11,003 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:16:11,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:13,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:15,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:15,662 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:16:17,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:19,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:21,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:23,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:25,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:26,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:16:26,223 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:16:27,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:29,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:31,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:33,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:35,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:37,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:39,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:41,437 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:16:41,438 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:16:41,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:43,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:45,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:45,736 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:16:47,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:49,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:51,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:53,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:55,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:56,711 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:16:56,711 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:16:57,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:16:59,631 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:01,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:03,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:05,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:07,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:09,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:11,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:11,935 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:17:11,935 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:17:13,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:15,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:15,810 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:17:17,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:19,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:22,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:24,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:26,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:27,215 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:17:27,215 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:17:28,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:30,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:32,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:34,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:36,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:38,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:40,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:42,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:17:42,477 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:17:42,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:44,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:45,882 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:17:46,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:48,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:50,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:52,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:54,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:56,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:17:57,770 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:17:57,771 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:17:58,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:00,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:02,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:04,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:06,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:08,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:10,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:12,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:13,023 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:18:13,023 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:18:14,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:15,953 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:18:16,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:18,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:20,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:22,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:24,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:26,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:28,253 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:18:28,254 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:18:28,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:30,673 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:32,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:34,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:36,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:38,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:40,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:42,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:43,469 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:18:43,470 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:18:44,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:46,029 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:18:46,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:48,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:50,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:52,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:54,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:56,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:58,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:18:58,860 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:18:58,860 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:19:00,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:03,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:05,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:07,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:09,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:11,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:13,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:14,080 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:19:14,081 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:19:15,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:16,108 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:19:17,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:19,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:21,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:23,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:25,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:27,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:29,294 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:19:29,294 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:19:29,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:31,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:33,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:35,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:37,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:39,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:41,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:43,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:44,683 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:19:44,684 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:19:45,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:46,182 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:19:47,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:49,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:51,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:53,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:55,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:57,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:59,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:19:59,909 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:19:59,909 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:20:01,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:03,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:05,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:07,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:09,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:11,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:13,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:15,188 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:20:15,188 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:20:15,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:16,260 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:20:17,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:19,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:21,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:23,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:25,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:28,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:30,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:20:30,445 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:20:30,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:32,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:34,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:36,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:38,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:40,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:42,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:44,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:45,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:20:45,839 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:20:46,334 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:20:46,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:48,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:50,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:52,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:54,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:56,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:20:58,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:00,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:01,063 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:21:01,064 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:21:02,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:04,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:06,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:08,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:10,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:12,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:14,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:16,360 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:21:16,360 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:21:16,583 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:21:16,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:18,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:20,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:22,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:24,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:26,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:28,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:30,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:31,618 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:21:31,619 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:21:32,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:34,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:36,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:38,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:40,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:42,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:44,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:46,483 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:21:46,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:46,839 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:21:46,839 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:21:48,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:50,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:52,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:55,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:57,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:21:59,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:01,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:02,149 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:22:02,149 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:22:03,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:05,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:07,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:09,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:11,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:13,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:15,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:16,561 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:22:17,467 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:22:17,467 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:22:17,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:19,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:21,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:23,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:25,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:27,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:29,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:31,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:32,744 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:22:32,744 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:22:33,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:35,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:37,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:39,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:41,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:43,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:45,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:46,632 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:22:47,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:47,967 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:22:47,968 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:22:49,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:51,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:53,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:55,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:57,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:22:59,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:01,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:03,193 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:23:03,194 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:23:03,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:05,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:07,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:09,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:11,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:14,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:16,698 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:23:16,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:18,552 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:23:18,553 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:23:18,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:20,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:22,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:24,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:26,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:28,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:30,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:32,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:33,838 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:23:33,839 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:23:34,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:36,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:38,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:40,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:42,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:44,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:46,777 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:23:46,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:48,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:49,106 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:23:49,107 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:23:50,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:52,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:54,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:56,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:23:58,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:00,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:02,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:04,348 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:24:04,348 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:24:04,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:06,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:08,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:10,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:12,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:14,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:16,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:16,853 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:24:18,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:19,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:24:19,569 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:24:20,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:22,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:24,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:26,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:28,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:30,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:32,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:34,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:34,845 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:24:34,845 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:24:36,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:39,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:41,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:43,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:45,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:46,925 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:24:47,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:49,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:50,067 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:24:50,067 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:24:51,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:53,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:55,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:57,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:24:59,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:01,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:03,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:05,290 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:25:05,291 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:25:05,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:07,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:09,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:11,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:13,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:15,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:17,001 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:25:17,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:19,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:20,524 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:25:20,524 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:25:21,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:23,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:25,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:27,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:29,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:31,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:33,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:35,739 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:25:35,740 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:25:35,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:37,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:39,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:41,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:43,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:45,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:47,077 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:25:47,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:49,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:51,017 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:25:51,017 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:25:51,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:53,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:55,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:57,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:25:59,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:01,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:03,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:05,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:06,366 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:26:06,366 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:26:07,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:09,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:11,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:14,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:16,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:17,153 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:26:18,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:20,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:21,589 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:26:21,590 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:26:22,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:24,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:26,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:28,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:30,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:32,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:34,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:36,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:36,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:26:36,925 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:26:38,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:40,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:42,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:44,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:46,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:47,226 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:26:48,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:50,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:52,259 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:26:52,260 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:26:52,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:54,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:56,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:26:58,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:00,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:02,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:04,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:06,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:07,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:27:07,480 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:27:08,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:10,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:12,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:14,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:16,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:17,301 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:27:19,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:21,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:22,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:27:22,699 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:27:23,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:25,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:27,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:29,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:31,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:33,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:35,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:37,917 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:27:37,918 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:27:37,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:39,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:41,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:43,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:45,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:47,376 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:27:47,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:49,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:51,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:53,206 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:27:53,207 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:27:53,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:55,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:57,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:27:59,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:01,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:03,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:05,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:07,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:08,425 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:28:08,425 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:28:09,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:11,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:13,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:15,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:17,449 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:28:17,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:19,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:21,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:23,649 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:28:23,649 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:28:23,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:25,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:27,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:29,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:31,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:33,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:35,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:37,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:38,864 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:28:38,864 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:28:39,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:42,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:44,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:46,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:47,526 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:28:48,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:50,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:52,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:54,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:28:54,244 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:28:54,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:56,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:28:58,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:00,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:02,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:04,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:06,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:08,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:09,477 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:29:09,477 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:29:10,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:12,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:14,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:16,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:17,604 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:29:18,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:20,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:22,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:24,798 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:29:24,798 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:29:24,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:26,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:28,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:30,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:32,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:34,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:36,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:38,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:40,019 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:29:40,020 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:29:40,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:42,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:44,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:46,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:47,678 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:29:48,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:50,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:52,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:54,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:55,242 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:29:55,242 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:29:56,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:29:58,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:00,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:02,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:04,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:06,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:08,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:10,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:30:10,525 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:30:10,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:12,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:14,989 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:16,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:17,754 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:30:18,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:20,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:22,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:25,741 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:30:25,741 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:30:25,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:27,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:29,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:31,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:33,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:35,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:38,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:40,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:41,125 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:30:41,125 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:30:42,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:44,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:46,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:47,830 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:30:48,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:50,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:52,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:54,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:56,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:30:56,452 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:30:56,453 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:30:58,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:00,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:02,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:04,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:06,013 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:08,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:10,015 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:11,687 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:31:11,687 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:31:12,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:14,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:16,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:17,906 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:31:18,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:20,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:22,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:24,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:26,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:26,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:31:26,909 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:31:28,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:30,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:32,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:34,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:36,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:38,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:40,027 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:42,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:42,213 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:31:42,214 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:31:44,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:46,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:47,983 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:31:48,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:50,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:52,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:54,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:57,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:31:57,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:31:57,484 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:31:59,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:01,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:03,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:05,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:07,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:09,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:11,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:12,777 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:32:12,778 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:32:13,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:15,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:17,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:18,059 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:32:19,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:21,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:23,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:25,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:27,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:28,044 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:32:28,044 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:32:29,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:31,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:33,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:35,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:37,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:39,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:42,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:43,266 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:32:43,267 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:32:44,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:46,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:48,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:48,132 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:32:50,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:52,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:54,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:56,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:58,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:32:58,617 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:32:58,617 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:33:00,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:02,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:04,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:06,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:08,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:10,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:12,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:13,927 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:33:13,928 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:33:14,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:16,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:18,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:18,207 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:33:20,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:22,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:24,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:26,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:28,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:29,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:33:29,151 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:33:30,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:32,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:34,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:36,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:38,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:40,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:42,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:44,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:44,542 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:33:44,543 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:33:46,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:48,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:48,288 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:33:50,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:52,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:54,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:56,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:58,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:33:59,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:33:59,788 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:34:00,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:02,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:05,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:07,092 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:09,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:11,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:13,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:15,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:34:15,081 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:34:15,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:17,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:18,381 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:34:19,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:21,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:23,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:25,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:27,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:29,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:30,423 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:34:30,423 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:34:31,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:33,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:35,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:37,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:39,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:41,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:43,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:45,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:45,648 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:34:45,648 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:34:47,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:48,435 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:34:49,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:51,112 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:53,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:55,114 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:57,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:34:59,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:01,008 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:35:01,008 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:35:01,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:03,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:05,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:07,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:09,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:11,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:13,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:15,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:16,230 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:35:16,230 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:35:17,124 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:18,509 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:35:19,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:21,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:24,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:26,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:28,129 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:30,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:31,565 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:35:31,565 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:35:32,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:34,132 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:36,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:38,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:40,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:42,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:44,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:46,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:46,788 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:35:46,788 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:35:48,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:48,585 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:35:50,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:52,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:54,140 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:56,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:35:58,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:00,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:02,074 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:36:02,075 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:36:02,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:04,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:06,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:08,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:10,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:12,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:14,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:16,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:17,297 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:36:17,297 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:36:18,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:18,655 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:36:20,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:22,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:24,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:26,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:28,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:30,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:32,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:32,512 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:36:32,513 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:36:34,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:36,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:38,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:40,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:42,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:44,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:46,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:47,897 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:36:47,897 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:36:48,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:48,731 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:36:50,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:52,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:54,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:56,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:36:58,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:00,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:02,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:03,132 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:37:03,132 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:37:04,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:06,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:08,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:11,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:13,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:15,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:17,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:18,350 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:37:18,350 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:37:18,805 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:37:19,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:21,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:23,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:25,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:27,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:29,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:31,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:33,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:33,624 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:37:33,625 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:37:35,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:37,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:39,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:41,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:43,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:45,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:47,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:48,882 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:37:48,908 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:37:48,908 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:37:49,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:51,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:53,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:55,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:57,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:37:59,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:01,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:03,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:04,120 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:38:04,121 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:38:05,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:07,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:09,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:11,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:13,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:15,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:17,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:18,958 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:38:19,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:19,461 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:38:19,462 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:38:21,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:23,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:25,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:27,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:29,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:31,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:33,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:34,764 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:38:34,765 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:38:36,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:38,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:40,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:42,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:44,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:46,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:48,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:49,038 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:38:50,014 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:38:50,015 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:38:50,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:52,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:54,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:56,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:38:58,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:00,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:02,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:04,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:05,282 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:39:05,283 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:39:06,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:08,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:10,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:12,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:14,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:16,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:18,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:19,114 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:39:20,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:20,509 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:39:20,510 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:39:22,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:24,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:26,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:28,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:30,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:32,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:34,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:35,817 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:39:35,817 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:39:36,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:38,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:40,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:42,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:44,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:46,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:48,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:49,192 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:39:50,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:52,205 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:39:52,205 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:39:52,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:54,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:56,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:39:58,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:00,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:02,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:04,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:06,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:07,608 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:40:07,608 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:40:09,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:11,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:13,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:15,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:17,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:19,250 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:19,270 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:40:21,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:22,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:40:22,829 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:40:23,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:25,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:27,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:29,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:31,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:33,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:35,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:37,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:38,051 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:40:38,052 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:40:39,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:41,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:43,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:45,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:47,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:49,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:49,346 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:40:51,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:53,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:53,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:40:53,378 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:40:55,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:57,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:40:59,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:01,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:03,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:05,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:07,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:08,600 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:41:08,600 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:41:09,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:11,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:13,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:15,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:17,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:19,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:19,421 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:41:21,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:23,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:23,840 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:41:23,840 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:41:25,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:27,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:29,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:31,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:33,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:35,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:37,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:39,090 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:41:39,090 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:41:39,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:41,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:43,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:45,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:47,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:49,496 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:41:50,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:52,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:54,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:54,441 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:41:54,441 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:41:56,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:41:58,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:00,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:02,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:04,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:06,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:08,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:09,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:42:09,747 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:42:10,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:12,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:14,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:16,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:18,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:19,570 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:42:20,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:22,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:24,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:25,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:42:25,032 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:42:26,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:28,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:30,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:32,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:34,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:36,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:38,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:40,256 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:42:40,256 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:42:40,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:42,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:44,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:46,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:48,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:49,643 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:42:50,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:52,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:54,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:55,658 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:42:55,659 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:42:56,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:42:58,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:00,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:02,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:04,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:06,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:08,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:10,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:10,877 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:43:10,877 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:43:12,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:14,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:16,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:18,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:19,715 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:43:20,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:22,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:24,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:26,119 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:43:26,119 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:43:26,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:28,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:30,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:32,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:34,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:36,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:38,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:40,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:41,356 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:43:41,356 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:43:42,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:44,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:46,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:49,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:49,781 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:43:51,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:53,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:55,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:56,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:43:56,572 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:43:57,337 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:43:59,338 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:01,339 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:03,340 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:05,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:07,341 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:09,342 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:11,343 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:11,849 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:44:11,849 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:44:13,344 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:15,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:17,345 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:19,346 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:19,855 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:44:21,347 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:23,348 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:25,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:27,152 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:44:27,152 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:44:27,349 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:29,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:31,351 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:33,352 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:35,353 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:37,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:39,354 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:41,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:42,372 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:44:42,372 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:44:43,356 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:45,357 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:47,358 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:49,359 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:49,931 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:44:51,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:53,365 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:55,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:57,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:44:57,707 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:44:57,707 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:44:59,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:01,368 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:03,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:05,370 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:07,371 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:09,372 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:11,373 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:13,374 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:14,559 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:45:14,559 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:45:15,375 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:17,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:19,377 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:20,008 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:45:21,378 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:23,379 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:25,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:27,380 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:29,381 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:29,932 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:45:29,932 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:45:31,382 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:33,383 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:35,384 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:37,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:39,385 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:41,386 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:43,387 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:45,222 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:45:45,222 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:45:46,389 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:48,390 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:50,082 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:45:50,391 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:52,392 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:54,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:56,393 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:45:58,394 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:00,395 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:00,531 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:46:00,532 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:46:02,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:04,397 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:06,398 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:08,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:10,399 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:12,400 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:14,401 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:15,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:46:15,824 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:46:16,402 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:18,403 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:20,156 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:46:20,404 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:22,405 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:24,406 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:26,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:28,407 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:30,408 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:31,047 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:46:31,047 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:46:32,409 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:34,410 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:36,411 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:38,412 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:40,413 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:42,414 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:44,415 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:46,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:46:46,365 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:46:46,416 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:48,417 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:50,223 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:46:50,418 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:52,419 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:54,420 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:56,421 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:46:58,422 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:00,423 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:01,760 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:47:01,760 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:47:02,424 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:04,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:06,425 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:08,426 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:10,427 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:12,428 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:14,429 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:16,430 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:16,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:47:16,980 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:47:18,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:20,295 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:47:20,431 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:22,432 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:24,433 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:26,434 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:28,435 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:30,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:32,201 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:47:32,201 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:47:32,436 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:35,437 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:37,438 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:39,439 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:41,440 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:43,441 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:45,442 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:47,443 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:47,540 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:47:47,540 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:47:49,444 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:50,370 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:47:51,445 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:53,446 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:55,447 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:57,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:47:59,448 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:01,449 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:02,835 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:48:02,835 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:48:03,450 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:05,451 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:07,452 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:09,453 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:11,454 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:13,455 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:15,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:17,456 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:18,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:48:18,049 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:48:19,457 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:20,435 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:48:21,458 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:23,459 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:25,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:27,460 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:29,461 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:31,462 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:33,338 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:48:33,339 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:48:33,463 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:35,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:37,464 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:39,465 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:41,466 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:43,467 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:45,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:47,468 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:48,623 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:48:48,623 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:48:49,469 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:50,511 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:48:51,470 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:54,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:56,472 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:48:58,473 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:00,474 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:02,475 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:03,861 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:49:03,861 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:49:04,476 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:06,477 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:08,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:10,478 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:12,479 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:14,480 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:16,481 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:18,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:19,238 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:49:19,238 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:49:20,482 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:20,579 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:49:22,483 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:24,484 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:26,485 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:28,486 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:30,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:32,487 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:34,488 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:34,564 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:49:34,564 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:49:36,489 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:38,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:40,490 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:42,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:44,491 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:46,492 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:48,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:49,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:49:49,885 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:49:50,493 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:50,655 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:49:52,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:54,494 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:56,495 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:49:58,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:00,496 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:02,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:04,497 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:05,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:50:05,147 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:50:06,498 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:08,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:10,499 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:12,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:14,500 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:16,501 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:18,502 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:20,364 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:50:20,365 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:50:20,503 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:20,730 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:50:23,504 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:25,505 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:27,506 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:29,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:31,507 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:33,508 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:35,509 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:35,757 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:50:35,757 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:50:37,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:39,510 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:41,511 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:43,512 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:45,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:47,513 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:49,514 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:50,805 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:50:51,039 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:50:51,039 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:50:51,515 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:53,516 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:55,517 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:57,518 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:50:59,519 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:01,520 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:03,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:05,521 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:06,289 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:51:06,289 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:51:07,522 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:09,523 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:11,524 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:13,525 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:15,526 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:17,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:19,527 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:20,881 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:51:21,528 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:21,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:51:21,610 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:51:23,529 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:25,530 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:27,531 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:29,532 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:31,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:33,533 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:35,534 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:36,880 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:51:36,880 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:51:37,535 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:40,536 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:42,537 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:44,538 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:46,539 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:48,540 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:50,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:50,954 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:51:52,113 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:51:52,113 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:51:52,541 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:54,542 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:56,543 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:51:58,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:00,544 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:02,545 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:04,546 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:06,547 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:07,447 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:52:07,448 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:52:08,548 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:10,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:12,549 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:14,550 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:16,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:18,551 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:20,552 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:21,032 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:52:22,553 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:22,676 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:52:22,677 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:52:24,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:26,554 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:28,555 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:30,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:32,556 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:34,557 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:36,558 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:37,983 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:52:37,983 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:52:38,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:40,559 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:42,560 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:44,561 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:46,562 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:48,563 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:50,564 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:51,106 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:52:52,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:53,204 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:52:53,204 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:52:54,565 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:56,566 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:52:58,567 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:00,568 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:02,569 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:04,570 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:06,571 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:08,511 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:53:08,511 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:53:08,572 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:10,573 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:12,574 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:14,575 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:16,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:18,576 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:20,577 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:21,179 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:53:22,578 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:23,745 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:53:23,745 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:53:24,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:26,579 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:28,580 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:30,581 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:32,582 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:34,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:36,583 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:38,584 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:38,962 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:53:38,962 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:53:40,585 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:42,586 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:44,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:46,587 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:49,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:51,249 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:53:51,589 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:53,590 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:54,308 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:53:54,308 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:53:55,591 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:57,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:53:59,592 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:01,593 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:03,594 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:05,595 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:07,596 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:09,528 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:54:09,528 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:54:09,597 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:11,598 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:13,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:15,599 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:17,600 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:19,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:21,323 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:54:21,602 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:23,603 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:24,885 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:54:24,885 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:54:25,604 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:27,605 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:29,606 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:31,607 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:33,608 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:35,609 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:37,610 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:39,611 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:40,116 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:54:40,116 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:54:41,612 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:43,613 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:45,614 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:47,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:49,615 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:51,399 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:54:51,616 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:53,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:55,445 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:54:55,446 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:54:55,617 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:57,618 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:54:59,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:01,619 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:03,620 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:05,621 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:07,622 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:09,623 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:10,814 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:55:10,815 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:55:11,624 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:13,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:15,625 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:17,626 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:19,627 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:21,473 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:55:21,628 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:23,629 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:25,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:26,034 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:55:26,034 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:55:27,630 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:30,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:32,632 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:34,633 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:36,634 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:38,635 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:40,636 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:41,722 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:55:41,723 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:55:42,637 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:44,638 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:46,639 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:48,640 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:50,641 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:51,548 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:55:52,642 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:54,643 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:56,644 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:55:57,029 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:55:57,030 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:55:58,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:00,645 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:02,646 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:04,647 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:06,648 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:08,649 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:10,650 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:12,285 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:56:12,285 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:56:12,651 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:14,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:16,652 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:18,653 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:20,654 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:21,626 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:56:22,655 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:24,656 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:26,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:27,585 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:56:27,586 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:56:28,657 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:30,658 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:32,659 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:34,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:36,660 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:38,661 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:40,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:42,662 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:42,808 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:56:42,809 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:56:44,663 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:47,664 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:49,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:51,665 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:51,697 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:56:53,666 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:55,667 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:57,668 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:56:58,031 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:56:58,031 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:56:59,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:01,669 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:03,670 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:05,671 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:07,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:09,672 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:11,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:13,365 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:57:13,365 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:57:13,674 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:15,675 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:17,676 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:19,677 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:21,678 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:21,760 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:57:23,679 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:25,680 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:27,681 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:28,636 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:57:28,636 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:57:29,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:31,682 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:33,683 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:35,684 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:37,685 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:39,686 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:41,687 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:43,688 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:43,856 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:57:43,856 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:57:45,689 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:47,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:49,690 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:51,691 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:51,836 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:57:53,692 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:55,693 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:57,694 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:57:59,189 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:57:59,189 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:57:59,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:01,695 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:04,696 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:06,697 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:08,698 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:10,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:12,699 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:14,486 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:58:14,486 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:58:14,700 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:16,701 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:18,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:20,702 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:21,909 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:58:22,703 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:24,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:26,704 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:28,705 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:29,710 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:58:29,711 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:58:30,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:32,706 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:34,707 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:36,708 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:38,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:40,709 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:42,710 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:44,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:45,098 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:58:45,099 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:58:46,711 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:48,712 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:50,713 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:51,988 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:58:52,714 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:54,715 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:56,716 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:58:58,717 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:00,539 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:59:00,539 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:59:00,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:02,718 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:04,719 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:06,720 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:08,721 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:10,722 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:12,723 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:14,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:15,778 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:59:15,778 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:59:16,724 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:18,725 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:20,726 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:22,061 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:59:22,727 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:24,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:26,728 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:28,729 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:30,730 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:31,004 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:59:31,005 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:59:32,731 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:34,732 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:37,733 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:39,734 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:41,735 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:43,736 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:45,737 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:46,220 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 19:59:46,220 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 19:59:47,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:49,738 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:51,739 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:52,134 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 19:59:53,740 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:55,741 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:57,742 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 19:59:59,743 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:01,535 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:00:01,535 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:00:01,744 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:03,745 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:05,746 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:07,747 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:09,748 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:11,749 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:13,750 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:15,751 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:16,758 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:00:16,759 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:00:17,752 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:19,753 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:21,754 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:22,193 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:00:23,755 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:25,756 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:27,757 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:29,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:31,758 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:32,071 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:00:32,071 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:00:33,759 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:35,760 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:37,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:39,761 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:41,762 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:43,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:45,763 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:47,378 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:00:47,379 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:00:47,764 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:49,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:51,765 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:52,260 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:00:53,766 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:55,767 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:57,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:00:59,768 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:01,769 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:02,599 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:01:02,599 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:01:03,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:05,770 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:07,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:09,771 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:11,772 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:13,773 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:16,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:17,910 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:01:17,910 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:01:18,774 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:20,775 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:22,331 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:01:22,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:24,776 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:26,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:28,777 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:30,778 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:32,779 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:33,131 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:01:33,132 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:01:34,780 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:36,781 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:38,782 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:40,783 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:42,784 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:44,785 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:46,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:48,352 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:01:48,352 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:01:48,786 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:50,787 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:52,403 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:01:52,788 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:54,789 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:56,790 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:01:58,791 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:00,792 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:02,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:03,671 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:02:03,671 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:02:04,793 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:06,794 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:08,795 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:11,796 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:13,797 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:15,798 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:17,799 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:18,892 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:02:18,892 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:02:19,800 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:21,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:22,479 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:02:23,801 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:25,802 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:27,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:29,803 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:31,804 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:33,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:34,112 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:02:34,113 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:02:35,805 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:37,806 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:39,807 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:41,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:43,808 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:45,809 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:47,810 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:49,466 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:02:49,467 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:02:49,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:51,811 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:52,553 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:02:53,812 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:55,813 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:57,814 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:02:59,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:01,815 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:03,816 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:04,691 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:03:04,691 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:03:05,817 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:07,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:09,818 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:11,819 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:13,820 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:15,821 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:17,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:19,822 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:19,916 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:03:19,916 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:03:21,823 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:22,630 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:03:23,824 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:25,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:27,825 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:29,826 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:31,827 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:33,828 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:35,130 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:03:35,131 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:03:35,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:37,829 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:39,830 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:41,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:43,831 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:45,832 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:47,833 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:49,834 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:50,346 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:03:50,346 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:03:51,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:52,705 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:03:53,835 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:55,836 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:57,837 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:03:59,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:01,838 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:03,839 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:05,699 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:04:05,699 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:04:06,840 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:08,841 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:10,842 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:12,843 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:14,844 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:16,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:18,845 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:20,846 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:20,922 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:04:20,923 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:04:22,779 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:04:22,847 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:24,848 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:26,849 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:28,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:30,850 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:32,851 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:34,852 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:36,211 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:04:36,211 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:04:36,853 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:38,854 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:40,855 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:42,856 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:44,857 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:46,858 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:48,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:50,859 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:51,553 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:04:51,553 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:04:52,855 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:04:52,860 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:54,861 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:56,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:04:58,862 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:00,863 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:02,864 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:04,865 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:06,823 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:05:06,824 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:05:06,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:08,866 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:10,867 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:12,868 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:14,869 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:16,870 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:18,871 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:20,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:22,092 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:05:22,093 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:05:22,872 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:22,927 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:05:24,873 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:26,874 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:28,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:30,875 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:32,876 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:34,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:36,877 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:37,312 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:05:37,312 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:05:38,878 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:40,879 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:42,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:44,880 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:46,881 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:48,882 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:50,883 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:52,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:05:52,554 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:05:52,884 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:53,000 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:05:54,885 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:56,886 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:05:58,887 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:00,888 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:03,889 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:05,890 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:07,774 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:06:07,774 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:06:07,891 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:09,892 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:11,893 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:13,894 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:15,895 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:17,896 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:19,897 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:21,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:23,067 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:06:23,077 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:06:23,077 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:06:23,898 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:25,899 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:27,900 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:29,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:31,901 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:33,902 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:35,903 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:37,904 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:38,330 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:06:38,331 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:06:39,905 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:41,906 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:43,907 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:45,908 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:47,909 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:49,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:51,910 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:53,142 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:06:53,554 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:06:53,555 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:06:53,911 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:55,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:57,912 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:06:59,913 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:01,914 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:03,915 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:05,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:07,916 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:08,866 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:07:08,866 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:07:09,917 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:11,918 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:13,919 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:15,920 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:17,921 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:19,922 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:21,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:23,218 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:07:23,923 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:24,146 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:07:24,146 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:07:25,924 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:27,925 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:29,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:31,926 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:33,927 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:35,928 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:37,929 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:39,453 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:07:39,454 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:07:39,930 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:42,931 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:44,932 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:46,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:48,933 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:50,934 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:52,935 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:53,295 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:07:54,738 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:07:54,738 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:07:54,936 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:56,937 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:07:58,938 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:00,939 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:02,940 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:04,941 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:06,942 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:08,943 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:09,999 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:08:09,999 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:08:10,944 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:12,945 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:14,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:16,946 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:18,947 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:20,948 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:22,949 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:23,374 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:08:24,950 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:25,274 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:08:25,275 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:08:26,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:28,951 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:30,952 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:32,953 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:34,954 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:36,955 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:38,956 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:40,533 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:08:40,533 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:08:40,957 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:42,958 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:44,959 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:47,960 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:49,961 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:51,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:53,452 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:08:53,962 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:55,829 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:08:55,829 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:08:55,963 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:57,964 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:08:59,965 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:01,966 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:03,967 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:05,968 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:07,969 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:09,970 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:11,971 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:13,263 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:09:13,263 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:09:13,972 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:15,973 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:17,974 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:19,975 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:21,976 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:23,527 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:09:23,977 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:25,978 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:27,979 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:28,566 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:09:28,567 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:09:29,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:31,980 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:33,981 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:35,982 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:37,983 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:39,984 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:41,985 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:43,782 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:09:43,783 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:09:43,986 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:45,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:47,987 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:50,988 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:52,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:53,607 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:09:54,990 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:56,991 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:58,992 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:09:59,026 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:09:59,026 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:10:00,993 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:02,994 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:04,995 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:06,996 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:08,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:10,997 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:12,998 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:14,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:10:14,244 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:10:14,999 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:17,000 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:19,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:21,001 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:23,002 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:23,682 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:10:25,003 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:27,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:29,004 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:29,591 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:10:29,591 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:10:31,005 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:33,006 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:35,007 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:37,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:39,008 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:41,009 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:43,010 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:44,852 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:10:44,852 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:10:45,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:47,011 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:49,012 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:51,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:53,014 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:53,757 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:10:55,016 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:57,017 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:10:59,018 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:00,136 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:11:00,136 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:11:01,019 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:03,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:05,020 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:07,021 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:09,022 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:11,023 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:13,024 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:15,025 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:15,363 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:11:15,364 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:11:17,026 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:20,028 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:22,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:23,832 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:11:24,029 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:26,030 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:28,031 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:30,032 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:30,590 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:11:30,590 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:11:32,033 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:34,034 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:36,035 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:38,036 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:40,037 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:42,038 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:44,039 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:45,937 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:11:45,938 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:11:46,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:48,040 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:50,041 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:52,042 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:53,909 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:11:54,043 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:56,044 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:11:58,045 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:00,046 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:01,750 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:12:01,750 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:12:02,047 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:04,048 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:06,049 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:08,050 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:10,051 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:12,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:14,052 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:16,053 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:17,086 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:12:17,086 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:12:18,054 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:20,055 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:22,056 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:23,980 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:12:24,057 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:26,058 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:28,059 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:30,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:32,060 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:32,347 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:12:32,347 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:12:34,061 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:36,062 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:38,063 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:40,064 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:42,065 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:44,066 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:47,067 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:47,568 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:12:47,569 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:12:49,068 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:51,069 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:53,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:54,043 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:12:55,070 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:57,071 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:12:59,072 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:01,073 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:02,872 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:13:02,872 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:13:03,074 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:05,075 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:07,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:09,076 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:11,077 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:13,078 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:15,079 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:17,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:18,095 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:13:18,095 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:13:19,080 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:21,081 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:23,082 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:24,116 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:13:25,083 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:27,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:29,084 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:31,085 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:33,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:33,354 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:13:33,354 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:13:35,086 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:37,087 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:39,088 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:41,089 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:43,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:45,090 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:47,091 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:48,572 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:13:48,573 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:13:50,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:52,093 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:54,094 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:54,192 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:13:56,095 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:13:58,096 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:00,097 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:02,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:03,894 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:14:03,895 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:14:04,098 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:06,099 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:08,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:10,101 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:12,102 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:14,103 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:16,104 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:18,105 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:19,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:14:19,109 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:14:20,106 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:22,107 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:24,108 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:24,267 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:14:26,109 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:28,110 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:30,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:32,111 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:34,112 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:34,336 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:14:34,337 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:14:36,113 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:38,114 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:40,115 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:42,116 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:44,117 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:46,118 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:48,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:49,654 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:14:49,655 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:14:50,119 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:52,120 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:54,121 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:54,337 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:14:56,122 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:14:58,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:00,123 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:02,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:04,125 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:04,875 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:15:04,875 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:15:06,126 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:08,127 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:11,128 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:13,130 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:15,131 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:17,132 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:19,133 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:20,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:15:20,141 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:15:21,134 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:23,135 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:24,408 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:15:25,136 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:27,137 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:29,138 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:31,139 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:33,140 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:35,141 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:35,505 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:15:35,505 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:15:37,142 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:39,143 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:41,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:43,144 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:45,145 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:47,146 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:49,147 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:50,803 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:15:50,804 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:15:51,148 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:53,149 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:54,481 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:15:55,150 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:57,151 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:15:59,152 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:01,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:03,153 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:05,154 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:06,109 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:16:06,109 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:16:07,155 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:09,156 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:11,157 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:13,158 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:15,159 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:17,160 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:19,161 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:21,162 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:21,331 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:16:21,331 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:16:23,163 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:24,545 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:16:25,164 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:27,165 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:29,166 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:31,167 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:33,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:35,168 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:36,627 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:16:36,627 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:16:37,169 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:39,170 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:41,171 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:43,172 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:45,173 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:47,174 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:49,175 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:51,176 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:51,918 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:16:51,919 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:16:54,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:54,622 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:16:56,177 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:16:58,178 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:00,179 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:02,180 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:04,181 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:06,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:07,143 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:17:07,143 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:17:08,182 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:10,183 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:12,184 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:14,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:16,185 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:18,186 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:20,187 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:22,188 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:22,370 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:17:22,371 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:17:24,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:24,689 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:17:26,189 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:28,190 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:30,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:32,191 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:34,192 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:37,193 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:37,609 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:17:37,609 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:17:39,194 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:41,195 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:43,196 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:45,197 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:47,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:49,198 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:51,199 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:52,911 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:17:52,912 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:17:53,200 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:54,762 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:17:55,201 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:57,202 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:17:59,203 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:01,204 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:03,205 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:05,206 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:07,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:08,141 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:18:08,141 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:18:09,207 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:11,208 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:13,209 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:15,210 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:17,211 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:19,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:21,212 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:23,213 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:23,368 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:18:23,368 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:18:24,837 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:18:25,214 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:27,215 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:29,216 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:31,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:33,217 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:35,218 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:37,219 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:39,049 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:18:39,049 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:18:39,220 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:41,221 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:43,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:45,222 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:47,223 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:49,224 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:51,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:53,225 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:54,432 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:18:54,432 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:18:54,913 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:18:55,226 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:57,227 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:18:59,228 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:01,229 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:03,230 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:05,231 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:07,232 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:09,233 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:09,656 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:19:09,656 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:19:12,234 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:14,235 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:16,236 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:18,237 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:20,238 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:22,239 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:24,240 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:24,956 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:19:24,957 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:19:25,181 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:19:26,241 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:28,242 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:30,243 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:32,244 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:34,245 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:36,246 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:38,247 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:40,248 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:40,261 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:19:40,261 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:19:42,249 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:44,251 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:46,252 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:48,253 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:50,254 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:52,255 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:54,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:55,066 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:19:55,482 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:19:55,482 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:19:56,256 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:19:58,257 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:00,258 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:02,259 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:04,260 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:06,261 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:08,262 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:10,263 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:10,698 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:20:10,698 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:20:12,264 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:14,265 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:16,266 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:18,267 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:20,268 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:22,269 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:24,270 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:25,142 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:20:25,985 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:20:25,985 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:20:26,271 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:28,272 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:30,273 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:32,274 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:34,275 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:36,276 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:38,277 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:40,278 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:41,250 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:20:41,250 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:20:42,279 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:44,280 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:46,281 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:48,282 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:50,283 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:52,284 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:54,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:55,218 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:20:56,285 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:20:56,507 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:20:56,507 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:20:58,286 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:00,287 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:02,288 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:04,289 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:06,290 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:08,291 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:10,292 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:11,729 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:21:11,729 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:21:12,293 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:14,294 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:16,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:18,295 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:20,296 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:22,297 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:24,298 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:25,294 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:21:26,299 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:27,081 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:21:27,082 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:21:28,300 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:30,301 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:32,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:34,302 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:36,303 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:39,304 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:41,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:42,417 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:21:42,417 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:21:43,305 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:45,306 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:47,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:49,307 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:51,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:53,308 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:55,309 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:55,355 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:21:57,310 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:21:57,715 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:21:57,715 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:21:59,311 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:01,312 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:03,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:05,313 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:07,314 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:09,315 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:11,316 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:12,963 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:22:12,963 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:22:13,317 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:15,318 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:17,319 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:19,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:21,320 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:23,321 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:25,322 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:25,421 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:22:27,323 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:28,244 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:22:28,244 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:22:29,324 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:31,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:33,325 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:35,326 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:37,327 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:39,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:41,328 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:43,329 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:43,483 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:22:43,483 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:22:45,330 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:48,331 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:50,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:52,332 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:54,333 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:55,494 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:22:56,334 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:58,335 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:22:58,749 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:22:58,749 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:23:00,336 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:23:13,974 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:23:13,974 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:23:25,571 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:23:29,227 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:23:29,227 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:23:44,489 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:23:44,489 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:23:55,647 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:23:59,748 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:23:59,748 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:24:15,005 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:24:15,005 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:24:17,980 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: partial_history -2022-08-30 20:24:17,986 DEBUG SenderThread:3760704 [sender.py:send():232] send: history -2022-08-30 20:24:18,033 DEBUG SenderThread:3760704 [sender.py:send():232] send: summary -2022-08-30 20:24:18,098 INFO SenderThread:3760704 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-08-30 20:24:18,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/wandb-summary.json -2022-08-30 20:24:19,366 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:24:20,367 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:24:24,369 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:24:25,725 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:24:30,302 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:24:30,304 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:24:40,376 INFO Thread-7 :3760704 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20220830_110431-yvlr8ud4/files/output.log -2022-08-30 20:24:45,525 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:24:45,526 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:24:55,804 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:25:00,747 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:25:00,748 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:25:15,970 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:25:15,970 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:25:25,882 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:25:31,195 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:25:31,196 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:25:46,429 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:25:46,429 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:25:55,961 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:26:01,809 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:26:01,810 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:26:17,036 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:26:17,036 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:26:26,040 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:26:32,258 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:26:32,259 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:26:47,480 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:26:47,481 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:26:56,117 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:27:02,702 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:27:02,702 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:27:17,925 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:27:17,926 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:27:26,198 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:27:33,150 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:27:33,151 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:27:48,373 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:27:48,374 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:27:56,274 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats -2022-08-30 20:28:03,598 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:28:03,598 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:28:18,827 DEBUG HandlerThread:3760704 [handler.py:handle_request():141] handle_request: stop_status -2022-08-30 20:28:18,828 DEBUG SenderThread:3760704 [sender.py:send_request():246] send_request: stop_status -2022-08-30 20:28:26,350 DEBUG SenderThread:3760704 [sender.py:send():232] send: stats +version https://git-lfs.github.com/spec/v1 +oid sha256:b37fc3daf18498771183630ef95b2ba2d4d769bafad7790ad0aea3511436b180 +size 10878553